Skip to content

Commit 0aebbad

Browse files
65436543
committed
Resolve AWS config on startup and add instance type deploy fallbacks
Port of #637 onto current main: - aws-instance-type becomes a string slice, entries in 'type' or 'type:region' form, tried in order as deploy fallbacks - AMI and instance types are resolved and validated at startup (arch match, region offering) - deprecated aws-user-data handling was already removed on main (#633), the port keeps the new RenderUserDataTemplate signature
1 parent d1358b4 commit 0aebbad

3 files changed

Lines changed: 233 additions & 32 deletions

File tree

‎providers/aws/flags.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,9 @@ const Category = "AWS"
66

77
var ProviderFlags = []cli.Flag{
88
// aws
9-
&cli.StringFlag{
9+
&cli.StringSliceFlag{
1010
Name: "aws-instance-type",
11-
Usage: "EC2 instance type",
11+
Usage: "EC2 instance types, optionally with region as 'type:region' (e.g. 't3.micro:us-east-1'); tried in order as deploy fallbacks. Omit the region to let AWS decide",
1212
Sources: cli.EnvVars("WOODPECKER_AWS_INSTANCE_TYPE"),
1313
Category: Category,
1414
},

‎providers/aws/helper.go‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
package aws
2+
3+
import (
4+
"context"
5+
"errors"
6+
"fmt"
7+
"strings"
8+
9+
"github.com/aws/aws-sdk-go-v2/aws"
10+
"github.com/aws/aws-sdk-go-v2/service/ec2"
11+
ec2_types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
12+
"github.com/aws/smithy-go"
13+
14+
"go.woodpecker-ci.org/woodpecker/v3/woodpecker-go/woodpecker"
15+
)
16+
17+
// resolveDeployCandidates parses the aws-instance-type entries (each in
18+
// "type" or "type:region" form) into ordered deploy candidates. An entry
19+
// without a region lets AWS pick one at deploy time.
20+
func (p *provider) resolveDeployCandidates(ctx context.Context, instanceTypes []string) error {
21+
for _, raw := range instanceTypes {
22+
rawType, region, _ := strings.Cut(raw, ":")
23+
24+
it, err := p.resolveInstanceType(ctx, rawType)
25+
if err != nil {
26+
return err
27+
}
28+
29+
// The AMI has a single architecture; an instance type whose
30+
// architectures don't include it can never boot this image.
31+
if !instanceTypeSupportsArch(it, p.image.Architecture) {
32+
return fmt.Errorf("%s: %w: %s needs one of %v, AMI is %s",
33+
p.name, ErrArchMismatch, it.InstanceType,
34+
it.ProcessorInfo.SupportedArchitectures, p.image.Architecture)
35+
}
36+
37+
if region != "" {
38+
if err := p.checkTypeOfferedInRegion(ctx, rawType, region); err != nil {
39+
return err
40+
}
41+
}
42+
43+
p.deployCandidates = append(p.deployCandidates, deployCandidate{
44+
instanceType: it,
45+
region: region,
46+
})
47+
}
48+
49+
if len(p.deployCandidates) == 0 {
50+
return fmt.Errorf("%s: %w", p.name, ErrNoDeployCandidates)
51+
}
52+
53+
return nil
54+
}
55+
56+
func (p *provider) resolveInstanceType(ctx context.Context, instanceType string) (ec2_types.InstanceTypeInfo, error) {
57+
out, err := p.client.DescribeInstanceTypes(ctx, &ec2.DescribeInstanceTypesInput{
58+
InstanceTypes: []ec2_types.InstanceType{ec2_types.InstanceType(instanceType)},
59+
})
60+
if err != nil {
61+
return ec2_types.InstanceTypeInfo{}, fmt.Errorf("%s: DescribeInstanceTypes %q: %w", p.name, instanceType, err)
62+
}
63+
if len(out.InstanceTypes) == 0 {
64+
return ec2_types.InstanceTypeInfo{}, fmt.Errorf("%s: %w: %s", p.name, ErrInstanceTypeNotFound, instanceType)
65+
}
66+
return out.InstanceTypes[0], nil
67+
}
68+
69+
// checkTypeOfferedInRegion verifies the instance type is actually offered in
70+
// the requested region, querying that region directly.
71+
func (p *provider) checkTypeOfferedInRegion(ctx context.Context, instanceType, region string) error {
72+
out, err := p.client.DescribeInstanceTypeOfferings(ctx, &ec2.DescribeInstanceTypeOfferingsInput{
73+
LocationType: ec2_types.LocationTypeRegion,
74+
Filters: []ec2_types.Filter{
75+
{Name: aws.String("instance-type"), Values: []string{instanceType}},
76+
{Name: aws.String("location"), Values: []string{region}},
77+
},
78+
}, func(o *ec2.Options) { o.Region = region })
79+
if err != nil {
80+
return fmt.Errorf("%s: DescribeInstanceTypeOfferings %q: %w", p.name, instanceType, err)
81+
}
82+
if len(out.InstanceTypeOfferings) == 0 {
83+
return fmt.Errorf("%s: %w: %s in %s", p.name, ErrTypeNotInRegion, instanceType, region)
84+
}
85+
return nil
86+
}
87+
88+
func instanceTypeSupportsArch(it ec2_types.InstanceTypeInfo, arch ec2_types.ArchitectureValues) bool {
89+
if it.ProcessorInfo == nil {
90+
return false
91+
}
92+
for _, a := range it.ProcessorInfo.SupportedArchitectures {
93+
if string(a) == string(arch) {
94+
return true
95+
}
96+
}
97+
return false
98+
}
99+
100+
func (p *provider) resolveImage(ctx context.Context, amiID string) error {
101+
out, err := p.client.DescribeImages(ctx, &ec2.DescribeImagesInput{
102+
ImageIds: []string{amiID},
103+
})
104+
if err != nil {
105+
return fmt.Errorf("%s: DescribeImages %q: %w", p.name, amiID, err)
106+
}
107+
if len(out.Images) == 0 {
108+
return fmt.Errorf("%s: %w: %s", p.name, ErrAMINotFound, amiID)
109+
}
110+
p.image = out.Images[0]
111+
return nil
112+
}
113+
114+
func (p *provider) getAgent(ctx context.Context, agent *woodpecker.Agent) (*ec2_types.Instance, error) {
115+
instances, err := p.client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{
116+
Filters: []ec2_types.Filter{
117+
{
118+
Name: aws.String("tag:Name"),
119+
Values: []string{agent.Name},
120+
},
121+
},
122+
})
123+
if err != nil {
124+
return nil, err
125+
}
126+
if len(instances.Reservations) != 1 {
127+
return nil, fmt.Errorf("expected 1 reservation with tag:Name=%s, got %d", agent.Name, len(instances.Reservations))
128+
}
129+
if len(instances.Reservations[0].Instances) != 1 {
130+
return nil, fmt.Errorf("expected 1 instance with tag:Name=%s, got %d", agent.Name, len(instances.Reservations[0].Instances))
131+
}
132+
return &instances.Reservations[0].Instances[0], nil
133+
}
134+
135+
// isInsufficientCapacity reports whether err is an AWS capacity error, for
136+
// which deploying the next fallback candidate is worthwhile.
137+
func isInsufficientCapacity(err error) bool {
138+
var apiErr smithy.APIError
139+
if errors.As(err, &apiErr) {
140+
return apiErr.ErrorCode() == "InsufficientInstanceCapacity"
141+
}
142+
return false
143+
}

‎providers/aws/provider.go‎

Lines changed: 88 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package aws
33
import (
44
"context"
55
b64 "encoding/base64"
6+
"errors"
67
"fmt"
78
"strings"
89
"sync"
@@ -22,11 +23,25 @@ import (
2223
"go.woodpecker-ci.org/woodpecker/v3/woodpecker-go/woodpecker"
2324
)
2425

26+
var (
27+
ErrInstanceTypeNotFound = errors.New("instance type not found")
28+
ErrAMINotFound = errors.New("AMI not found")
29+
ErrSubnetsNotSet = errors.New("aws-subnets must be set")
30+
ErrArchMismatch = errors.New("instance type architecture not supported by AMI")
31+
ErrTypeNotInRegion = errors.New("instance type not offered in region")
32+
ErrNoDeployCandidates = errors.New("no deploy candidates resolved")
33+
)
34+
35+
// deployCandidate is one resolved instance-type/region pair the provider will
36+
// try, in order, when deploying an agent. An empty region lets AWS decide.
37+
type deployCandidate struct {
38+
instanceType ec2_types.InstanceTypeInfo
39+
region string
40+
}
41+
2542
type provider struct {
2643
name string
2744
config *config.Config
28-
instanceType string
29-
amiID string
3045
tags []string
3146
region string
3247
subnets []string
@@ -37,17 +52,18 @@ type provider struct {
3752
lock sync.Mutex
3853
subnetRR int
3954
sshKeyName string
55+
// resolved config
56+
deployCandidates []deployCandidate
57+
image ec2_types.Image
4058
}
4159

4260
func New(ctx context.Context, c *cli.Command, config *config.Config) (types.Provider, error) {
4361
if len(c.StringSlice("aws-subnets")) == 0 {
44-
return nil, fmt.Errorf("aws-subnets must be set")
62+
return nil, ErrSubnetsNotSet
4563
}
4664
p := &provider{
4765
name: "aws",
4866
config: config,
49-
instanceType: c.String("aws-instance-type"),
50-
amiID: c.String("aws-ami-id"),
5167
tags: c.StringSlice("aws-tags"),
5268
region: c.String("aws-region"),
5369
subnets: c.StringSlice("aws-subnets"),
@@ -62,9 +78,38 @@ func New(ctx context.Context, c *cli.Command, config *config.Config) (types.Prov
6278
}
6379
p.client = ec2.NewFromConfig(cfg)
6480

81+
// AMI must be resolved first: its architecture constrains which instance
82+
// types are valid deploy candidates.
83+
if err := p.resolveImage(ctx, c.String("aws-ami-id")); err != nil {
84+
return nil, err
85+
}
86+
if err := p.resolveDeployCandidates(ctx, c.StringSlice("aws-instance-type")); err != nil {
87+
return nil, err
88+
}
89+
90+
p.printResolvedConfig()
91+
6592
return p, nil
6693
}
6794

95+
func (p *provider) printResolvedConfig() {
96+
log.Info().
97+
Str("ami", aws.ToString(p.image.ImageId)).
98+
Str("ami_arch", string(p.image.Architecture)).
99+
Msg("resolved AMI")
100+
for _, c := range p.deployCandidates {
101+
region := c.region
102+
if region == "" {
103+
region = "<aws-decides>"
104+
}
105+
log.Info().
106+
Str("type", string(c.instanceType.InstanceType)).
107+
Str("region", region).
108+
Bool("current_gen", aws.ToBool(c.instanceType.CurrentGeneration)).
109+
Msg("deploy candidate")
110+
}
111+
}
112+
68113
func (p *provider) DeployAgent(ctx context.Context, agent *woodpecker.Agent) error {
69114
userData, err := cloudinit.RenderUserDataTemplate(p.config, agent, cloudinit.RenderOption{})
70115
if err != nil {
@@ -103,8 +148,7 @@ func (p *provider) DeployAgent(ctx context.Context, agent *woodpecker.Agent) err
103148
IamInstanceProfile: &ec2_types.IamInstanceProfileSpecification{
104149
Arn: aws.String(p.iamInstanceProfileArn),
105150
},
106-
ImageId: aws.String(p.amiID),
107-
InstanceType: ec2_types.InstanceType(p.instanceType),
151+
ImageId: p.image.ImageId,
108152
MetadataOptions: &ec2_types.InstanceMetadataOptionsRequest{
109153
HttpEndpoint: ec2_types.InstanceMetadataEndpointStateEnabled,
110154
HttpPutResponseHopLimit: aws.Int32(1),
@@ -142,8 +186,43 @@ func (p *provider) DeployAgent(ctx context.Context, agent *woodpecker.Agent) err
142186
}
143187

144188
runInstancesInput.UserData = aws.String(b64.StdEncoding.EncodeToString([]byte(userData)))
145-
result, err := p.client.RunInstances(ctx, &runInstancesInput)
146-
if err != nil {
189+
190+
var result *ec2.RunInstancesOutput
191+
for i, c := range p.deployCandidates {
192+
runInstancesInput.InstanceType = c.instanceType.InstanceType
193+
194+
// An empty region keeps the client's default region.
195+
var optFns []func(*ec2.Options)
196+
if c.region != "" {
197+
region := c.region
198+
optFns = append(optFns, func(o *ec2.Options) { o.Region = region })
199+
}
200+
201+
log.Info().
202+
Str("type", string(c.instanceType.InstanceType)).
203+
Str("region", c.region).
204+
Msg("create agent")
205+
206+
result, err = p.client.RunInstances(ctx, &runInstancesInput, optFns...)
207+
if err == nil {
208+
break
209+
}
210+
211+
// Continue to next fallback entry only if capacity is unavailable.
212+
if !isInsufficientCapacity(err) {
213+
return fmt.Errorf("%s: RunInstances: %w", p.name, err)
214+
}
215+
216+
// Only log and continue if there are more candidates left.
217+
if i < len(p.deployCandidates)-1 {
218+
log.Warn().Msgf(
219+
"create agent failed: type = %s region = %s: %s",
220+
c.instanceType.InstanceType, c.region, err,
221+
)
222+
continue
223+
}
224+
225+
// Last candidate failed.
147226
return fmt.Errorf("%s: RunInstances: %w", p.name, err)
148227
}
149228

@@ -169,27 +248,6 @@ func (p *provider) DeployAgent(ctx context.Context, agent *woodpecker.Agent) err
169248
return fmt.Errorf("instance did not resolve in agent list: %s", *result.Instances[0].InstanceId)
170249
}
171250

172-
func (p *provider) getAgent(ctx context.Context, agent *woodpecker.Agent) (*ec2_types.Instance, error) {
173-
instances, err := p.client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{
174-
Filters: []ec2_types.Filter{
175-
{
176-
Name: aws.String("tag:Name"),
177-
Values: []string{agent.Name},
178-
},
179-
},
180-
})
181-
if err != nil {
182-
return nil, err
183-
}
184-
if len(instances.Reservations) != 1 {
185-
return nil, fmt.Errorf("expected 1 reservation with tag:Name=%s, got %d", agent.Name, len(instances.Reservations))
186-
}
187-
if len(instances.Reservations[0].Instances) != 1 {
188-
return nil, fmt.Errorf("expected 1 instance with tag:Name=%s, got %d", agent.Name, len(instances.Reservations[0].Instances))
189-
}
190-
return &instances.Reservations[0].Instances[0], nil
191-
}
192-
193251
func (p *provider) RemoveAgent(ctx context.Context, agent *woodpecker.Agent) error {
194252
instance, err := p.getAgent(ctx, agent)
195253
if err != nil {

0 commit comments

Comments
 (0)