# nolint start
library(mlexperiments)
library(mllrnrs)The hardware and bandwidth for this mirror is donated by dogado GmbH, the Webhosting and Full Service-Cloud Provider. Check out our Wordpress Tutorial.
If you wish to report a bug, or if you are interested in having us mirror your free-software or open-source project, please feel free to contact us at mirror[@]dogado.de.
# nolint start
library(mlexperiments)
library(mllrnrs)See https://github.com/kapsner/mllrnrs/blob/main/R/learner_xgboost.R for implementation details.
library(mlbench)
data("DNA")
dataset <- DNA |>
data.table::as.data.table() |>
na.omit()
feature_cols <- colnames(dataset)[160:180]
target_col <- "Class"seed <- 123
if (isTRUE(as.logical(Sys.getenv("_R_CHECK_LIMIT_CORES_")))) {
# on cran
ncores <- 2L
} else {
ncores <- ifelse(
test = parallel::detectCores() > 4,
yes = 4L,
no = ifelse(
test = parallel::detectCores() < 2L,
yes = 1L,
no = parallel::detectCores()
)
)
}
options("mlexperiments.bayesian.max_init" = 4L)
options("mlexperiments.optim.xgb.nrounds" = 20L)
options("mlexperiments.optim.xgb.early_stopping_rounds" = 5L)data_split <- splitTools::partition(
y = dataset[, get(target_col)],
p = c(train = 0.7, test = 0.3),
type = "stratified",
seed = seed
)
train_x <- model.matrix(
~ -1 + .,
dataset[data_split$train, .SD, .SDcols = feature_cols]
)
train_y <- as.integer(dataset[data_split$train, get(target_col)]) - 1L
test_x <- model.matrix(
~ -1 + .,
dataset[data_split$test, .SD, .SDcols = feature_cols]
)
test_y <- as.integer(dataset[data_split$test, get(target_col)]) - 1Lfold_list <- splitTools::create_folds(
y = train_y,
k = 3,
type = "stratified",
seed = seed
)# required learner arguments, not optimized
learner_args <- list(
objective = "multi:softprob",
eval_metric = "mlogloss",
num_class = 3
)
# set arguments for predict function and performance metric,
# required for mlexperiments::MLCrossValidation and
# mlexperiments::MLNestedCV
predict_args <- list(reshape = TRUE)
performance_metric <- metric("ACC")
performance_metric_args <- NULL
return_models <- FALSE
# required for grid search and initialization of bayesian optimization
parameter_grid <- expand.grid(
subsample = seq(0.6, 1, .2),
colsample_bytree = seq(0.6, 1, .2),
min_child_weight = seq(1, 5, 4),
learning_rate = seq(0.1, 0.2, 0.1),
max_depth = seq(1, 5, 4)
)
# reduce to a maximum of 10 rows
if (nrow(parameter_grid) > 10) {
set.seed(123)
sample_rows <- sample(seq_len(nrow(parameter_grid)), 10, FALSE)
parameter_grid <- kdry::mlh_subset(parameter_grid, sample_rows)
}
# required for bayesian optimization
parameter_bounds <- list(
subsample = c(0.2, 1),
colsample_bytree = c(0.2, 1),
min_child_weight = c(1L, 10L),
learning_rate = c(0.1, 0.2),
max_depth = c(1L, 10L)
)
optim_args <- list(
n_iter = ncores,
kappa = 3.5,
acq = "ucb"
)tuner <- mlexperiments::MLTuneParameters$new(
learner = mllrnrs::LearnerXgboost$new(
metric_optimization_higher_better = FALSE
),
strategy = "grid",
ncores = ncores,
seed = seed
)
tuner$parameter_grid <- parameter_grid
tuner$learner_args <- learner_args
tuner$split_type <- "stratified"
tuner$set_data(
x = train_x,
y = train_y
)
tuner_results_grid <- tuner$execute(k = 3)
#>
#> Parameter settings [============================>-------------------------------------------------------------------] 3/10 ( 30%)
#> Parameter settings [=====================================>----------------------------------------------------------] 4/10 ( 40%)
#> Parameter settings [===============================================>------------------------------------------------] 5/10 ( 50%)
#> Parameter settings [=========================================================>--------------------------------------] 6/10 ( 60%)
#> Parameter settings [==================================================================>-----------------------------] 7/10 ( 70%)
#> Parameter settings [============================================================================>-------------------] 8/10 ( 80%)
#> Parameter settings [=====================================================================================>----------] 9/10 ( 90%)
#> Parameter settings [===============================================================================================] 10/10 (100%)
head(tuner_results_grid)
#> setting_id metric_optim_mean nrounds subsample colsample_bytree min_child_weight learning_rate max_depth objective
#> <int> <num> <int> <num> <num> <num> <num> <num> <char>
#> 1: 1 1.0106675 38 0.6 0.8 5 0.2 1 multi:softprob
#> 2: 2 0.9828797 37 1.0 0.8 5 0.1 5 multi:softprob
#> 3: 3 1.0102800 76 0.8 0.8 5 0.1 1 multi:softprob
#> 4: 4 0.9867769 20 0.6 0.8 5 0.2 5 multi:softprob
#> 5: 5 0.9815158 32 1.0 0.8 1 0.1 5 multi:softprob
#> 6: 6 0.9741743 50 0.8 0.8 5 0.1 5 multi:softprob
#> eval_metric num_class
#> <char> <num>
#> 1: mlogloss 3
#> 2: mlogloss 3
#> 3: mlogloss 3
#> 4: mlogloss 3
#> 5: mlogloss 3
#> 6: mlogloss 3tuner <- mlexperiments::MLTuneParameters$new(
learner = mllrnrs::LearnerXgboost$new(
metric_optimization_higher_better = FALSE
),
strategy = "bayesian",
ncores = ncores,
seed = seed
)
tuner$parameter_grid <- parameter_grid
tuner$parameter_bounds <- parameter_bounds
tuner$learner_args <- learner_args
tuner$optim_args <- optim_args
tuner$split_type <- "stratified"
tuner$set_data(
x = train_x,
y = train_y
)
tuner_results_bayesian <- tuner$execute(k = 3)
#>
#> Registering parallel backend using 4 cores.
head(tuner_results_bayesian)
#> Epoch setting_id subsample colsample_bytree min_child_weight learning_rate max_depth gpUtility acqOptimum inBounds Elapsed
#> <num> <int> <num> <num> <num> <num> <num> <num> <lgcl> <lgcl> <num>
#> 1: 0 1 0.6 0.8 5 0.2 1 NA FALSE TRUE 0.995
#> 2: 0 2 1.0 0.8 5 0.1 5 NA FALSE TRUE 1.057
#> 3: 0 3 0.8 0.8 5 0.1 1 NA FALSE TRUE 1.088
#> 4: 0 4 0.6 0.8 5 0.2 5 NA FALSE TRUE 1.035
#> 5: 0 5 1.0 0.8 1 0.1 5 NA FALSE TRUE 0.282
#> 6: 0 6 0.8 0.8 5 0.1 5 NA FALSE TRUE 0.314
#> Score metric_optim_mean nrounds errorMessage objective eval_metric num_class
#> <num> <num> <int> <lgcl> <char> <char> <num>
#> 1: -1.0106675 1.0106675 38 NA multi:softprob mlogloss 3
#> 2: -0.9828797 0.9828797 37 NA multi:softprob mlogloss 3
#> 3: -1.0102800 1.0102800 76 NA multi:softprob mlogloss 3
#> 4: -0.9867769 0.9867769 20 NA multi:softprob mlogloss 3
#> 5: -0.9815158 0.9815158 32 NA multi:softprob mlogloss 3
#> 6: -0.9741743 0.9741743 50 NA multi:softprob mlogloss 3validator <- mlexperiments::MLCrossValidation$new(
learner = mllrnrs::LearnerXgboost$new(
metric_optimization_higher_better = FALSE
),
fold_list = fold_list,
ncores = ncores,
seed = seed
)
validator$learner_args <- tuner$results$best.setting[-1]
validator$predict_args <- predict_args
validator$performance_metric <- performance_metric
validator$performance_metric_args <- performance_metric_args
validator$return_models <- return_models
validator$set_data(
x = train_x,
y = train_y
)
validator_results <- validator$execute()
#>
#> CV fold: Fold1
#>
#> CV fold: Fold2
#>
#> CV fold: Fold3
#> CV progress [========================================================================================================] 3/3 (100%)
#>
head(validator_results)
#> fold performance subsample colsample_bytree min_child_weight learning_rate max_depth nrounds objective eval_metric
#> <char> <num> <num> <num> <num> <num> <num> <int> <char> <char>
#> 1: Fold1 0.5685484 0.7220588 0.8330246 1 0.1099728 10 19 multi:softprob mlogloss
#> 2: Fold2 0.5587045 0.7220588 0.8330246 1 0.1099728 10 19 multi:softprob mlogloss
#> 3: Fold3 0.5483871 0.7220588 0.8330246 1 0.1099728 10 19 multi:softprob mlogloss
#> num_class
#> <num>
#> 1: 3
#> 2: 3
#> 3: 3validator <- mlexperiments::MLNestedCV$new(
learner = mllrnrs::LearnerXgboost$new(
metric_optimization_higher_better = FALSE
),
strategy = "grid",
fold_list = fold_list,
k_tuning = 3L,
ncores = ncores,
seed = seed
)
validator$parameter_grid <- parameter_grid
validator$learner_args <- learner_args
validator$split_type <- "stratified"
validator$predict_args <- predict_args
validator$performance_metric <- performance_metric
validator$performance_metric_args <- performance_metric_args
validator$return_models <- return_models
validator$set_data(
x = train_x,
y = train_y
)
validator_results <- validator$execute()
#>
#> CV fold: Fold1
#>
#> Parameter settings [============================>-------------------------------------------------------------------] 3/10 ( 30%)
#> Parameter settings [=====================================>----------------------------------------------------------] 4/10 ( 40%)
#> Parameter settings [===============================================>------------------------------------------------] 5/10 ( 50%)
#> Parameter settings [=========================================================>--------------------------------------] 6/10 ( 60%)
#> Parameter settings [==================================================================>-----------------------------] 7/10 ( 70%)
#> Parameter settings [============================================================================>-------------------] 8/10 ( 80%)
#> Parameter settings [=====================================================================================>----------] 9/10 ( 90%)
#> Parameter settings [===============================================================================================] 10/10 (100%)
#> CV fold: Fold2
#> CV progress [====================================================================>-----------------------------------] 2/3 ( 67%)
#>
#> Parameter settings [============================>-------------------------------------------------------------------] 3/10 ( 30%)
#> Parameter settings [=====================================>----------------------------------------------------------] 4/10 ( 40%)
#> Parameter settings [===============================================>------------------------------------------------] 5/10 ( 50%)
#> Parameter settings [=========================================================>--------------------------------------] 6/10 ( 60%)
#> Parameter settings [==================================================================>-----------------------------] 7/10 ( 70%)
#> Parameter settings [============================================================================>-------------------] 8/10 ( 80%)
#> Parameter settings [=====================================================================================>----------] 9/10 ( 90%)
#> Parameter settings [===============================================================================================] 10/10 (100%)
#> CV fold: Fold3
#> CV progress [========================================================================================================] 3/3 (100%)
#>
#> Parameter settings [============================>-------------------------------------------------------------------] 3/10 ( 30%)
#> Parameter settings [=====================================>----------------------------------------------------------] 4/10 ( 40%)
#> Parameter settings [===============================================>------------------------------------------------] 5/10 ( 50%)
#> Parameter settings [=========================================================>--------------------------------------] 6/10 ( 60%)
#> Parameter settings [==================================================================>-----------------------------] 7/10 ( 70%)
#> Parameter settings [============================================================================>-------------------] 8/10 ( 80%)
#> Parameter settings [=====================================================================================>----------] 9/10 ( 90%)
#> Parameter settings [===============================================================================================] 10/10 (100%)
head(validator_results)
#> fold performance nrounds subsample colsample_bytree min_child_weight learning_rate max_depth objective eval_metric
#> <char> <num> <int> <num> <num> <num> <num> <num> <char> <char>
#> 1: Fold1 0.5591398 32 0.6 1.0 1 0.1 5 multi:softprob mlogloss
#> 2: Fold2 0.5344130 34 0.8 0.8 5 0.1 5 multi:softprob mlogloss
#> 3: Fold3 0.5510753 24 0.6 1.0 1 0.1 5 multi:softprob mlogloss
#> num_class
#> <num>
#> 1: 3
#> 2: 3
#> 3: 3validator <- mlexperiments::MLNestedCV$new(
learner = mllrnrs::LearnerXgboost$new(
metric_optimization_higher_better = FALSE
),
strategy = "bayesian",
fold_list = fold_list,
k_tuning = 3L,
ncores = ncores,
seed = seed
)
validator$parameter_grid <- parameter_grid
validator$learner_args <- learner_args
validator$split_type <- "stratified"
validator$parameter_bounds <- parameter_bounds
validator$optim_args <- optim_args
validator$predict_args <- predict_args
validator$performance_metric <- performance_metric
validator$performance_metric_args <- performance_metric_args
validator$return_models <- TRUE
validator$set_data(
x = train_x,
y = train_y
)
validator_results <- validator$execute()
#>
#> CV fold: Fold1
#>
#> Registering parallel backend using 4 cores.
#>
#> CV fold: Fold2
#> CV progress [====================================================================>-----------------------------------] 2/3 ( 67%)
#>
#> Registering parallel backend using 4 cores.
#>
#> CV fold: Fold3
#> CV progress [========================================================================================================] 3/3 (100%)
#>
#> Registering parallel backend using 4 cores.
head(validator_results)
#> fold performance subsample colsample_bytree min_child_weight learning_rate max_depth nrounds objective eval_metric
#> <char> <num> <num> <num> <num> <num> <num> <int> <char> <char>
#> 1: Fold1 0.5591398 0.6000000 1.0000000 1 0.1000000 5 32 multi:softprob mlogloss
#> 2: Fold2 0.5641026 0.6936177 0.7695365 1 0.1099728 10 20 multi:softprob mlogloss
#> 3: Fold3 0.5537634 0.5955781 0.8688622 1 0.1099728 10 19 multi:softprob mlogloss
#> num_class
#> <num>
#> 1: 3
#> 2: 3
#> 3: 3preds_xgboost <- mlexperiments::predictions(
object = validator,
newdata = test_x
)perf_xgboost <- mlexperiments::performance(
object = validator,
prediction_results = preds_xgboost,
y_ground_truth = test_y
)
perf_xgboost
#> model performance
#> <char> <num>
#> 1: Fold1 0.5590387
#> 2: Fold2 0.5579937
#> 3: Fold3 0.5611285Here, xgboost’s weight-argument is used to rescale the case-weights during the training.
# define the target weights
y_weights <- ifelse(train_y == 1, 0.8, ifelse(train_y == 2, 1.2, 1))
head(y_weights)
#> [1] 1.2 1.2 0.0 0.8 0.8 0.0tuner_w_weights <- mlexperiments::MLTuneParameters$new(
learner = mllrnrs::LearnerXgboost$new(
metric_optimization_higher_better = FALSE
),
strategy = "grid",
ncores = ncores,
seed = seed
)
tuner_w_weights$parameter_grid <- parameter_grid
tuner_w_weights$learner_args <- c(
learner_args,
list(case_weights = y_weights)
)
tuner_w_weights$split_type <- "stratified"
tuner_w_weights$set_data(
x = train_x,
y = train_y
)
tuner_results_grid <- tuner_w_weights$execute(k = 3)
#>
#> Parameter settings [============================>-------------------------------------------------------------------] 3/10 ( 30%)
#> Parameter settings [=====================================>----------------------------------------------------------] 4/10 ( 40%)
#> Parameter settings [===============================================>------------------------------------------------] 5/10 ( 50%)
#> Parameter settings [=========================================================>--------------------------------------] 6/10 ( 60%)
#> Parameter settings [==================================================================>-----------------------------] 7/10 ( 70%)
#> Parameter settings [============================================================================>-------------------] 8/10 ( 80%)
#> Parameter settings [=====================================================================================>----------] 9/10 ( 90%)
#> Parameter settings [===============================================================================================] 10/10 (100%)
head(tuner_results_grid)
#> setting_id metric_optim_mean nrounds subsample colsample_bytree min_child_weight learning_rate max_depth objective
#> <int> <num> <int> <num> <num> <num> <num> <num> <char>
#> 1: 1 0.9442324 50 0.6 0.8 5 0.2 1 multi:softprob
#> 2: 2 0.9217258 35 1.0 0.8 5 0.1 5 multi:softprob
#> 3: 3 0.9443002 93 0.8 0.8 5 0.1 1 multi:softprob
#> 4: 4 0.9245540 20 0.6 0.8 5 0.2 5 multi:softprob
#> 5: 5 0.9212009 26 1.0 0.8 1 0.1 5 multi:softprob
#> 6: 6 0.9145242 39 0.8 0.8 5 0.1 5 multi:softprob
#> eval_metric num_class
#> <char> <num>
#> 1: mlogloss 3
#> 2: mlogloss 3
#> 3: mlogloss 3
#> 4: mlogloss 3
#> 5: mlogloss 3
#> 6: mlogloss 3validator_w_weights <- mlexperiments::MLCrossValidation$new(
learner = mllrnrs::LearnerXgboost$new(
metric_optimization_higher_better = FALSE
),
fold_list = fold_list,
ncores = ncores,
seed = seed
)
# append the optimized setting from above with the newly created weights
validator_w_weights$learner_args <- c(
tuner_w_weights$results$best.setting[-1]
)
validator_w_weights$predict_args <- predict_args
validator_w_weights$performance_metric <- performance_metric
validator_w_weights$performance_metric_args <- performance_metric_args
validator_w_weights$return_models <- return_models
validator_w_weights$set_data(
x = train_x,
y = train_y
)
validator_results <- validator_w_weights$execute()
#>
#> CV fold: Fold1
#>
#> CV fold: Fold2
#>
#> CV fold: Fold3
#> CV progress [========================================================================================================] 3/3 (100%)
#>
head(validator_results)
#> fold performance nrounds subsample colsample_bytree min_child_weight learning_rate max_depth objective eval_metric
#> <char> <num> <int> <num> <num> <num> <num> <num> <char> <char>
#> 1: Fold1 0.5658602 39 0.8 0.8 5 0.1 5 multi:softprob mlogloss
#> 2: Fold2 0.5222672 39 0.8 0.8 5 0.1 5 multi:softprob mlogloss
#> 3: Fold3 0.5577957 39 0.8 0.8 5 0.1 5 multi:softprob mlogloss
#> num_class
#> <num>
#> 1: 3
#> 2: 3
#> 3: 3These binaries (installable software) and packages are in development.
They may not be fully stable and should be used with caution. We make no claims about them.
Health stats visible at Monitor.