Rewrite scheduler to respect labels and capabilities of providers

.woodpecker/test.yaml

@@ -43,3 +43,10 @@ steps:
     commands:
       - make test
     when: *when
+
+  - name: test-e2e
+    image: *golang_image
+    depends_on: [vendor]
+    commands:
+      - make test-e2e
+    when: *when

Makefile

@@ -118,6 +118,13 @@ lint: install-tools ## Lint code
 test-autoscaler: ## Test autoscaler code
 	go test -race -cover -coverprofile autoscaler-coverage.out -timeout 30s ${GO_PACKAGES}
 
+.PHONY: test-e2e
+test-e2e: ## Run end-to-end tests and report the engine coverage they exercise
+	go test -race -coverpkg=go.woodpecker-ci.org/autoscaler/engine/... -coverprofile e2e-coverage.out -timeout 60s go.woodpecker-ci.org/autoscaler/e2e
+	@echo
+	@echo "e2e coverage of engine:"
+	@go tool cover -func=e2e-coverage.out
+
 .PHONY: test
 test: test-autoscaler ## Run all tests
 

README.md

@@ -100,4 +100,4 @@ The billing model is selected automatically by the provider, so no extra configu
   - [ ] Remove agents which have not connected for a long time
 - [x] Release as container image
 - [x] Add docs
-- [ ] Support agent deployment with specific attributes (e.g. platforms, architectures, etc.)
+- [x] Support agent deployment with specific attributes (e.g. platforms, architectures, etc.)

cmd/woodpecker-autoscaler/flags.go

@@ -44,6 +44,12 @@ var flags = []cli.Flag{
 		Usage:   "time an agent is allowed to be inactive before it can be terminated as duration string like 2h45m (https://pkg.go.dev/time#ParseDuration)",
 		Sources: cli.EnvVars("WOODPECKER_AGENT_INACTIVITY_TIMEOUT", "WOODPECKER_AGENT_ALLOWED_STARTUP_TIME"),
 	},
+	&cli.StringFlag{
+		Name:    "agent-creation-timeout",
+		Value:   "10m",
+		Usage:   "time a deployed agent is allowed to take for its first contact with the server before it is replaced and torn down, as duration string like 2h45m (https://pkg.go.dev/time#ParseDuration)",
+		Sources: cli.EnvVars("WOODPECKER_AGENT_CREATION_TIMEOUT", "WOODPECKER_AGENT_ALLOWED_STARTUP_TIME"),
+	},
 	&cli.StringFlag{
 		Name:    "agent-idle-timeout",
 		Value:   "10m",

cmd/woodpecker-autoscaler/main.go

@@ -99,18 +99,26 @@ func run(ctx context.Context, cmd *cli.Command) error {
 	}
 	config.BillingModel = provider.BillingModel()
 
-	autoscaler := engine.NewAutoscaler(provider, client, config)
-
 	config.AgentInactivityTimeout, err = time.ParseDuration(cmd.String("agent-inactivity-timeout"))
 	if err != nil {
 		return fmt.Errorf("can't parse agent-inactivity-timeout: %w", err)
 	}
 
+	config.AgentCreationTimeout, err = time.ParseDuration(cmd.String("agent-creation-timeout"))
+	if err != nil {
+		return fmt.Errorf("can't parse agent-creation-timeout: %w", err)
+	}
+
 	config.AgentIdleTimeout, err = time.ParseDuration(cmd.String("agent-idle-timeout"))
 	if err != nil {
 		return fmt.Errorf("can't parse agent-idle-timeout: %w", err)
 	}
 
+	autoscaler, err := engine.NewAutoscaler(ctx, provider, client, config)
+	if err != nil {
+		return fmt.Errorf("could not create autoscaler: %w", err)
+	}
+
 	config.AgentBillingTeardownMargin, err = time.ParseDuration(cmd.String("agent-billing-teardown-margin"))
 	if err != nil {
 		return fmt.Errorf("can't parse agent-billing-teardown-margin: %w", err)

config/config.go

@@ -17,8 +17,14 @@ type Config struct {
 	GRPCSecure             bool
 	AgentInactivityTimeout time.Duration
 	AgentIdleTimeout       time.Duration
-	UserData               string // cloudinit template
-	ExtraAgentLabels       map[string]string
+	// AgentCreationTimeout is how long a deployed agent may take to make its
+	// first contact with the server. Past this deadline it no longer counts
+	// as bucket capacity and is reaped, so a stuck boot cannot stall the
+	// queue indefinitely. Zero disables the deadline; never-connected agents
+	// then fall back to AgentInactivityTimeout for reaping.
+	AgentCreationTimeout time.Duration
+	UserData             string // cloudinit template
+	ExtraAgentLabels     map[string]string
 
 	// BillingModel is taken from the selected provider and selects the teardown
 	// policy the engine applies to idle agents.