fix(one_se_rule): handle small archives and keep n_features an integer (#174)

Author: be-marc

NEWS.md

@@ -1,6 +1,7 @@
 # mlr3fselect (development version)
 
 * fix: `ArchiveAsyncFSelect` pushed results with the removed `rush::Rush$push_results()` method.
+* fix: The `mlr3fselect.one_se_rule` callback errored on archives with a single evaluation or with missing scores, and wrote the `n_features` column as a list column instead of an integer column (#174).
 * fix: `extract_inner_fselect_results()` added the `iteration` and `fselect_instance` columns to the result of the inner `FSelectInstance` by reference, which created a circular reference between the instance and its own result (#172).
 * fix: `fs("rfe")` and `fs("rfecv")` failed with an internal `data.table` error when `store_benchmark_result = FALSE` was set because the importance scores were read from the benchmark result of the archive (#169).
 * fix: `fs("rfecv", recursive = FALSE)` failed with an internal `data.table` error because the importance scores of all resampling iterations were written to a single archive row (#168).

R/mlr_callbacks.R

@@ -166,11 +166,13 @@ load_callback_one_se_rule = function() {
     on_optimization_end = function(callback, context) {
       archive = context$instance$archive
       data = as.data.table(archive)
-      data[, "n_features" := map(get("features"), length)]
+      data[, "n_features" := lengths(get("features"))]
 
       # standard error
+      # `sd()` is `NA` for a single evaluation, in this case the smallest feature set is selected
       y = data[[archive$cols_y]]
-      se = sd(y) / sqrt(length(y))
+      n = sum(!is.na(y))
+      se = if (n < 2L) 0 else sd(y, na.rm = TRUE) / sqrt(n)
 
       columns_to_keep = setdiff(names(context$instance$result), "x_domain")
       if (se == 0) {
@@ -179,9 +181,10 @@ load_callback_one_se_rule = function() {
           data[, columns_to_keep, with = FALSE][which.min(n_features)]
       } else {
         # select smallest future set within one standard error of the best
+        # `which()` drops feature sets without a score
         best_y = context$instance$result_y
         context$instance$.__enclos_env__$private$.result =
-          data[y > best_y - se & y < best_y + se, columns_to_keep, with = FALSE][which.min(n_features)]
+          data[which(y > best_y - se & y < best_y + se), columns_to_keep, with = FALSE][which.min(n_features)]
       }
     }
   )

tests/testthat/test_mlr_callbacks.R

@@ -52,6 +52,25 @@ test_that("one_se_rule callback works", {
   )
 
   expect_equal(instance$result_feature_set, c("x1", "x2", "x3"))
+  # the number of features is a scalar and not a list column
+  expect_integer(instance$result$n_features, len = 1)
+})
+
+test_that("one_se_rule callback works with a single evaluation", {
+  instance = fselect(
+    fselector = fs("random_search", batch_size = 1),
+    task = TEST_MAKE_TSK(),
+    learner = lrn("regr.rpart"),
+    resampling = rsmp("cv", folds = 3),
+    measures = msr("dummy"),
+    term_evals = 1,
+    callbacks = clbk("mlr3fselect.one_se_rule")
+  )
+
+  # the standard error of a single evaluation is `NA`
+  expect_data_table(instance$archive$data, nrows = 1)
+  expect_data_table(instance$result, nrows = 1)
+  expect_equal(instance$result$features[[1]], as.data.table(instance$archive)$features[[1]])
 })
 
 test_that("internal tuning callback works", {