The hardware and bandwidth for this mirror is donated by dogado GmbH, the Webhosting and Full Service-Cloud Provider. Check out our Wordpress Tutorial.
If you wish to report a bug, or if you are interested in having us mirror your free-software or open-source project, please feel free to contact us at mirror[@]dogado.de.

KNN: Binary Classification

# nolint start
library(mlexperiments)

See https://github.com/kapsner/mlexperiments/blob/main/R/learner_knn.R for implementation details.

Preprocessing

Import and Prepare Data

library(mlbench)
data("BreastCancer")
dataset <- BreastCancer |>
  data.table::as.data.table() |>
  na.omit()

seed <- 123
feature_cols <- colnames(dataset)[2:10]
target_col <- "Class"
to_num <- c(
  "Cl.thickness",
  "Cell.size",
  "Cell.shape",
  "Marg.adhesion",
  "Epith.c.size"
)
dataset[, (to_num) := lapply(.SD, as.numeric), .SDcols = to_num]

General Configurations

seed <- 123
if (isTRUE(as.logical(Sys.getenv("_R_CHECK_LIMIT_CORES_")))) {
  # on cran
  ncores <- 2L
} else {
  ncores <- ifelse(
    test = parallel::detectCores() > 4,
    yes = 4L,
    no = ifelse(
      test = parallel::detectCores() < 2L,
      yes = 1L,
      no = parallel::detectCores()
    )
  )
}
options("mlexperiments.bayesian.max_init" = 4L)

Generate Training- and Test Data

data_split <- splitTools::partition(
  y = dataset[, get(target_col)],
  p = c(train = 0.7, test = 0.3),
  type = "stratified",
  seed = seed
)

train_x <- model.matrix(
  ~ -1 + .,
  dataset[data_split$train, .SD, .SDcols = feature_cols]
)
train_y <- as.integer(dataset[data_split$train, get(target_col)]) - 1L


test_x <- model.matrix(
  ~ -1 + .,
  dataset[data_split$test, .SD, .SDcols = feature_cols]
)
test_y <- as.integer(dataset[data_split$test, get(target_col)]) - 1L

Generate Training Data Folds

fold_list <- splitTools::create_folds(
  y = train_y,
  k = 3,
  type = "stratified",
  seed = seed
)

Experiments

Prepare Experiments

# required learner arguments, not optimized
learner_args <- list(
  l = 2,
  test = parse(text = "fold_test$x"),
  use.all = FALSE
)

# set arguments for predict function and performance metric,
# required for mlexperiments::MLCrossValidation and
# mlexperiments::MLNestedCV
predict_args <- list(type = "response")
performance_metric <- metric("ACC")
performance_metric_args <- NULL
return_models <- FALSE

# required for grid search and initialization of bayesian optimization
parameter_grid <- expand.grid(
  k = seq(4, 68, 6)
)
# reduce to a maximum of 10 rows
if (nrow(parameter_grid) > 10) {
  set.seed(123)
  sample_rows <- sample(seq_len(nrow(parameter_grid)), 10, FALSE)
  parameter_grid <- kdry::mlh_subset(parameter_grid, sample_rows)
}

# required for bayesian optimization
parameter_bounds <- list(k = c(2L, 80L))
optim_args <- list(
  n_iter = ncores,
  kappa = 3.5,
  acq = "ucb"
)

Hyperparameter Tuning

tuner <- mlexperiments::MLTuneParameters$new(
  learner = LearnerKnn$new(),
  strategy = "grid",
  ncores = ncores,
  seed = seed
)

tuner$parameter_grid <- parameter_grid
tuner$learner_args <- learner_args
tuner$split_type <- "stratified"

tuner$set_data(
  x = train_x,
  y = train_y
)

tuner_results_grid <- tuner$execute(k = 3)

head(tuner_results_grid)
#>    setting_id metric_optim_mean     k     l use.all
#>         <int>             <num> <num> <num>  <lgcl>
#> 1:          1        0.03979364    16     2   FALSE
#> 2:          2        0.06281546    64     2   FALSE
#> 3:          3        0.03771031    10     2   FALSE
#> 4:          4        0.04393394    34     2   FALSE
#> 5:          5        0.05862242    58     2   FALSE
#> 6:          6        0.04393394    28     2   FALSE

Bayesian Optimization

tuner <- mlexperiments::MLTuneParameters$new(
  learner = LearnerKnn$new(),
  strategy = "bayesian",
  ncores = ncores,
  seed = seed
)

tuner$parameter_grid <- parameter_grid
tuner$parameter_bounds <- parameter_bounds

tuner$learner_args <- learner_args
tuner$optim_args <- optim_args

tuner$split_type <- "stratified"

tuner$set_data(
  x = train_x,
  y = train_y
)

tuner_results_bayesian <- tuner$execute(k = 3)
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#> elapsed = 0.016  Round = 1   k = 10.0000 Value = -0.03771031
#> elapsed = 0.02   Round = 2   k = 4.0000  Value = -0.04607001
#> elapsed = 0.014  Round = 3   k = 64.0000 Value = -0.06281546
#> elapsed = 0.013  Round = 4   k = 52.0000 Value = -0.06070576
#> elapsed = 0.012  Round = 5   k = 25.0000 Value = -0.0418506
#> elapsed = 0.015  Round = 6   k = 80.0000 Value = -0.05860932
#> elapsed = 0.011  Round = 7   k = 17.0000 Value = -0.03976727
#> elapsed = 0.013  Round = 8   k = 37.0000 Value = -0.04811371
#>
#>  Best Parameters Found:
#> Round = 1    k = 10.0000 Value = -0.03771031

head(tuner_results_bayesian)
#>    setting_id     k       Value     l use.all metric_optim_mean
#>         <int> <num>       <num> <num>  <lgcl>             <num>
#> 1:          1    10 -0.03771031     2   FALSE        0.03771031
#> 2:          2     4 -0.04607001     2   FALSE        0.04607001
#> 3:          3    64 -0.06281546     2   FALSE        0.06281546
#> 4:          4    52 -0.06070576     2   FALSE        0.06070576
#> 5:          5    25 -0.04185060     2   FALSE        0.04185060
#> 6:          6    80 -0.05860932     2   FALSE        0.05860932

k-Fold Cross Validation

validator <- mlexperiments::MLCrossValidation$new(
  learner = LearnerKnn$new(),
  fold_list = fold_list,
  ncores = ncores,
  seed = seed
)

validator$learner_args <- tuner$results$best.setting

validator$predict_args <- predict_args
validator$performance_metric <- performance_metric
validator$performance_metric_args <- performance_metric_args
validator$return_models <- return_models

validator$set_data(
  x = train_x,
  y = train_y
)

validator_results <- validator$execute()
#>
#> CV fold: Fold1
#>
#> CV fold: Fold2
#>
#> CV fold: Fold3

head(validator_results)
#>      fold performance     k     l use.all
#>    <char>       <num> <num> <num>  <lgcl>
#> 1:  Fold1   0.9746835    10     2   FALSE
#> 2:  Fold2   0.9496855    10     2   FALSE
#> 3:  Fold3   0.9625000    10     2   FALSE

Nested Cross Validation

validator <- mlexperiments::MLNestedCV$new(
  learner = LearnerKnn$new(),
  strategy = "grid",
  fold_list = fold_list,
  k_tuning = 3L,
  ncores = ncores,
  seed = seed
)

validator$parameter_grid <- parameter_grid
validator$learner_args <- learner_args
validator$split_type <- "stratified"

validator$predict_args <- predict_args
validator$performance_metric <- performance_metric
validator$performance_metric_args <- performance_metric_args
validator$return_models <- return_models

validator$set_data(
  x = train_x,
  y = train_y
)

validator_results <- validator$execute()
#>
#> CV fold: Fold1
#>
#> CV fold: Fold2
#>
#> CV fold: Fold3
#> CV progress [========================================================================================================] 3/3 (100%)
#>

head(validator_results)
#>      fold performance     k     l use.all
#>    <char>       <num> <num> <num>  <lgcl>
#> 1:  Fold1   0.9746835    10     2   FALSE
#> 2:  Fold2   0.9496855    10     2   FALSE
#> 3:  Fold3   0.9625000    10     2   FALSE

Inner Bayesian Optimization

validator <- mlexperiments::MLNestedCV$new(
  learner = LearnerKnn$new(),
  strategy = "bayesian",
  fold_list = fold_list,
  k_tuning = 3L,
  ncores = ncores,
  seed = seed
)

validator$parameter_grid <- parameter_grid
validator$learner_args <- learner_args
validator$split_type <- "stratified"


validator$parameter_bounds <- parameter_bounds
validator$optim_args <- optim_args

validator$predict_args <- predict_args
validator$performance_metric <- performance_metric
validator$performance_metric_args <- performance_metric_args
validator$return_models <- return_models

validator$set_data(
  x = train_x,
  y = train_y
)

validator_results <- validator$execute()
#>
#> CV fold: Fold1
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#> elapsed = 0.009  Round = 1   k = 10.0000 Value = -0.04699348
#> elapsed = 0.008  Round = 2   k = 4.0000  Value = -0.05639805
#> elapsed = 0.01   Round = 3   k = 64.0000 Value = -0.07523658
#> elapsed = 0.01   Round = 4   k = 52.0000 Value = -0.07209193
#> elapsed = 0.009  Round = 5   k = 25.0000 Value = -0.05331217
#> elapsed = 0.011  Round = 6   k = 80.0000 Value = -0.08152589
#> elapsed = 0.009  Round = 7   k = 17.0000 Value = -0.05016752
#> elapsed = 0.011  Round = 8   k = 37.0000 Value = -0.06271675
#>
#>  Best Parameters Found:
#> Round = 1    k = 10.0000 Value = -0.04699348
#>
#> CV fold: Fold2
#> CV progress [====================================================================>-----------------------------------] 2/3 ( 67%)
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#> elapsed = 0.009  Round = 1   k = 10.0000 Value = -0.02830189
#> elapsed = 0.009  Round = 2   k = 4.0000  Value = -0.0408805
#> elapsed = 0.01   Round = 3   k = 64.0000 Value = -0.07232704
#> elapsed = 0.01   Round = 4   k = 52.0000 Value = -0.05974843
#> elapsed = 0.01   Round = 5   k = 25.0000 Value = -0.03773585
#> elapsed = 0.01   Round = 6   k = 80.0000 Value = -0.08490566
#> elapsed = 0.01   Round = 7   k = 17.0000 Value = -0.02830189
#> elapsed = 0.01   Round = 8   k = 39.0000 Value = -0.05031447
#>
#>  Best Parameters Found:
#> Round = 1    k = 10.0000 Value = -0.02830189
#>
#> CV fold: Fold3
#> CV progress [========================================================================================================] 3/3 (100%)
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#> elapsed = 0.01   Round = 1   k = 10.0000 Value = -0.04117999
#> elapsed = 0.008  Round = 2   k = 4.0000  Value = -0.0442947
#> elapsed = 0.01   Round = 3   k = 64.0000 Value = -0.06004792
#> elapsed = 0.01   Round = 4   k = 52.0000 Value = -0.05687332
#> elapsed = 0.009  Round = 5   k = 27.0000 Value = -0.04432465
#> elapsed = 0.009  Round = 6   k = 17.0000 Value = -0.04432465
#> elapsed = 0.011  Round = 7   k = 80.0000 Value = -0.06951183
#> elapsed = 0.01   Round = 8   k = 38.0000 Value = -0.05372866
#>
#>  Best Parameters Found:
#> Round = 1    k = 10.0000 Value = -0.04117999

head(validator_results)
#>      fold performance     k     l use.all
#>    <char>       <num> <num> <num>  <lgcl>
#> 1:  Fold1   0.9746835    10     2   FALSE
#> 2:  Fold2   0.9496855    10     2   FALSE
#> 3:  Fold3   0.9625000    10     2   FALSE

These binaries (installable software) and packages are in development.
They may not be fully stable and should be used with caution. We make no claims about them.
Health stats visible at Monitor.