随机搜索贝叶斯搜索

python网格搜索优化参数Keras深度学习超参数优化官方手册Keras深度学习超参数优化手册-CSDN博客版超参数搜索不够高效这几大策略了解一下使用贝叶斯优化进行深度神经网络超参数优化

网格搜索

n_out):cols.append(df.shift(-i))#

put

valuesagg.dropna(inplaceTrue)return

agg.values#

sqrt(mean_squared_error(actual,

predicted))#

train_x.reshape((train_x.shape[0],

train_x.shape[1],

Sequential()model.add(Conv1D(filtersn_filters,

kernel_sizen_kernel,

n_features)))model.add(MaxPooling1D(pool_size2))model.add(Flatten())model.add(Dense(1))model.compile(lossmse,

optimizeradam)#

array(history[-n_input:]).reshape((1,

n_input,

predictionspredictions.append(yhat)#

add

f]configs.append(cfg)print(Total

configs:

下载数据集https://raw.githubusercontent.com/jbrownlee/Datasets/master/airline-passengers.csv

series

read_csv(airline-passengers.csv,

header0,

Sequential()model.add(Dense(12,

input_shape(8,),

activationrelu))model.add(Dense(1,

activationsigmoid))#

modelmodel.compile(lossbinary_crossentropy,

optimizeradam,

np.loadtxt(pima-indians-diabetes.csv,

delimiter,)

KerasClassifier(modelcreate_model,

verbose0)

grid_result.cv_results_[mean_test_score]

stds

grid_result.cv_results_[std_test_score]

params

grid_result.cv_results_[params]

for

param))更多参考https://machinelearningmastery.com/grid-search-hyperparameters-deep-learning-models-python-keras/随机搜索

Load

create_model(hparams1dvalue,hparams2dvalue,...hparamsndvalue):#

Model

KerasClassifier(build_fncreate_model)

Specify

RandomizedSearchCV(estimatormodel,

param_distributionsparam_dist,n_itern_iter_search,n_jobs,

cv,

random_search.cv_results_[mean_test_score]

stds

random_search.cv_results_[std_test_score]

params

random_search.cv_results_[params]

for

Sequential()model.add(Dense(units

hp.Int(dense-bot,

2)):model.add(Dense(unitshp.Int(dense_

str(i),

activationrelu))model.add(Dropout(hp.Choice(dropout_

str(i),

0.2])))model.add(Dense(10,activationsoftmax))hp_optimizerhp.Choice(Optimizer,

values[Adam,

kt.tuners.BayesianOptimization(model,seedrandom_seed,objectiveval_loss,max_trials30,directory.,project_nametuning-mlp)

train_y,

tuner_mlp.get_best_hyperparameters(1)[0]

print(Best

best_mlp_hyperparameters.values使用最优参数来训练模型model_mlp

Sequential()model_mlp.add(Dense(best_mlp_hyperparameters[dense-bot],

input_shape(784,),

range(best_mlp_hyperparameters[num_dense_layers]):model_mlp.add(Dense(unitsbest_mlp_hyperparameters[dense_

str(i)],

activationrelu))model_mlp.add(Dropout(ratebest_mlp_hyperparameters[dropout_

str(i)]))model_mlp.add(Dense(10,activationsoftmax))model_mlp.compile(optimizerbest_mlp_hyperparameters[Optimizer],

losscategorical_crossentropy,metrics[accuracy])

history_mlp

model_mlptuner_mlp.hypermodel.build(best_mlp_hyperparameters)

history_mlpmodel_mlp.fit(train_x,

train_y,

callbackscallback)效果测试mlp_test_loss,

mlp_test_acc

model_cnn.compile(optimizeradam,

metrics[accuracy])贝叶斯搜索超参数model

Sequential()model

2)):hp_paddinghp.Choice(padding_

str(i),

same])hp_filtershp.Choice(filters_

str(i),

64])model.add(Conv2D(hp_filters,

(3,

2)))model.add(Dropout(hp.Choice(dropout_

str(i),

0.2])))model.add(Flatten())hp_units

hp.Int(units,

kernel_initializerhe_uniform))model.add(Dense(10,activationsoftmax))hp_learning_rate

values[1e-2,

hp_optimizerhp.Choice(Optimizer,

values[Adam,

optimizerhp_optimizer,losscategorical_crossentropy,

kt.tuners.BayesianOptimization(model,objectiveval_loss,max_trials100,directory.,project_nametuning-cnn)采用最佳超参数训练模型model_cnn

Sequential()model_cnn.add(Input(shape(28,

28,

range(best_cnn_hyperparameters[num_blocks]):hp_paddingbest_cnn_hyperparameters[padding_

str(i)]hp_filtersbest_cnn_hyperparameters[filters_

str(i)]model_cnn.add(Conv2D(hp_filters,

(3,

1)))model_cnn.add(MaxPooling2D((2,

2)))model_cnn.add(Dropout(best_cnn_hyperparameters[dropout_

str(i)]))model_cnn.add(Flatten())

model_cnn.add(Dense(best_cnn_hyperparameters[units],

activationrelu,

kernel_initializerhe_uniform))model_cnn.add(Dense(10,activationsoftmax))model_cnn.compile(optimizerbest_cnn_hyperparameters[Optimizer],

metrics[accuracy])

print(model_cnn.summary())history_cnn

train_y,

nemo-大模型训练优化自动超参数搜索分析https://github.com/NVIDIA/NeMo-Framework-Launcher

Optuna深度学习超参数优化框架

下述代码指定了SGDClassifier分类器的参数alpha、max_iter

def

sklearn.datasets.load_iris()classes

valid_x,

sklearn.model_selection.train_test_split(iris.data,

iris.target,

trial.suggest_loguniform(alpha,

1e-5,

trial.suggest_int(max_iter,64,192,step64)loss

trial.suggest_categorical(loss,[hinge,log,perceptron])clf

sklearn.linear_model.SGDClassifier(alphaalpha,max_itermax_iter)#

下述代码指定了学习率learning_rate、优化器optimizer、神经元个数n_uint

def

trial.suggest_loguniform(learning_rate,

1e-5,

trial.suggest_categorical(optimizer,

[Adam,

sklearn.datasets.load_iris()classes

valid_x,

sklearn.model_selection.train_test_split(iris.data,

iris.target,

trial.suggest_loguniform(alpha,

1e-5,

trial.suggest_int(max_iter,64,192,step64)loss

trial.suggest_categorical(loss,[hinge,log,perceptron])clf

sklearn.linear_model.SGDClassifier(alphaalpha,max_itermax_iter)for

step

range(100):clf.partial_fit(train_x,

train_y,

classesclasses)intermediate_value

1.0

valid_y)trial.report(intermediate_value,

step)if

sklearn.datasets.load_iris()classes

valid_x,

sklearn.model_selection.train_test_split(iris.data,

iris.target,

trial.suggest_loguniform(alpha,

1e-5,

trial.suggest_int(max_iter,64,192,step64)loss

trial.suggest_categorical(loss,[hinge,log,perceptron])clf

sklearn.linear_model.SGDClassifier(alphaalpha,max_itermax_iter)for

step

range(100):clf.partial_fit(train_x,

train_y,

classesclasses)intermediate_value

1.0

valid_y)trial.report(intermediate_value,

step)if

optuna.create_study(storagepath,study_namefirst,pruneroptuna.pruners.MedianPruner())

#study

optuna.study.load_study(first,path)

n_trials20)

trial.params.items():print({}:{}.format(key,

value))#

optuna.visualization.plot_contour(study)#若不行请尝试

vis_path

optuna.visualization.plot_contour(study,params[n_layers,lr])

plotly.offline.plot(graph_cout,filenamevis_pathgraph_cout.html)plot_optimization_history(study)#若不行请尝试

vis_path

plot_optimization_history(study)

plotly.offline.plot(history,filenamevis_pathhistory.html)plot_intermediate_values(study)#若不行请尝试

vis_path

plot_intermediate_values(study)

plotly.offline.plot(intermed,filenamevis_pathintermed.html)plot_slice(study,

params[alpha,max_iter,loss])#若不行请尝试

vis_path

plotly.offline.plot(slices,filenamevis_pathslices.html)plot_parallel_coordinate(study,params[alpha,max_iter,loss])#若不行请尝试

vis_path

plot_parallel_coordinate(study)

plotly.offline.plot(paraller,filenamevis_pathparaller.html)nvidia

nemo大模型超参数优化框架