def masktraining_skip_2chan(path_train, path_valid, batch_size, epochs, path_save_model, path_weights_model, plot_name, option, reduction_divisor): # initializing #!/usr/bin/env python3 # -*- coding: utf8 -*- import ctypes import os os.environ['TF_CPP_MIN_LOG_LEVEL'] = '1' os.environ['TF_ENABLE_XLA'] = '1' os.environ['TF_ENABLE_AUTO_MIXED_PRECISION'] = '1' os.environ['TF_ENABLE_CUDNN_RNN_TENSOR_OP_MATH_FP32'] = '1' os.environ['TF_DISABLE_CUDNN_TENSOR_OP_MATH'] = '1' os.environ['TF_ENABLE_CUBLAS_TENSOR_OP_MATH_FP32'] = '1' import numpy as np import tensorflow as tf from tensorflow import keras from keras.layers import Add, Multiply, Input, Dense, Flatten, Dropout, Conv2D, MaxPooling2D, Conv2DTranspose, LeakyReLU, Reshape, Activation, BatchNormalization, UpSampling2D from keras.models import Model from keras.constraints import max_norm from keras.optimizers import Adam, SGD from keras.utils import to_categorical, normalize from keras.callbacks import ModelCheckpoint from keras import losses from keras import backend as K import matplotlib.pyplot as plt import random # checking for gpus and using it/them gpus = tf.config.experimental.list_physical_devices('GPU') if gpus: try: tf.config.experimental.set_virtual_device_configuration(gpus[0], [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=4096)]) except RuntimeError as e: print(e) # custom generator import from Generators.MaskGenerator3_realimag import DataGenerator # ERSETZEN! def count_files(dir): return len([1 for x in list(os.scandir(dir)) if x.is_file()]) len_train = count_files(path_train) len_valid = count_files(path_valid) # generators for train and validation data train_generator = DataGenerator(path_train, option, reduction_divisor, len_train, batch_size, True) valid_generator = DataGenerator(path_valid, option, reduction_divisor, len_valid, batch_size, True) # building model def build_model(): max_norm_value = 100.0 # for first channel # defining inputs and normalizing just noisy input input_noisy_1 = Input(shape=(260,5,1)) input_noise_1 = Input(shape=(260,1,1)) input_speech_1 = Input(shape=(260,1,1)) normalized_noisy_1 = BatchNormalization()(input_noisy_1) # normalized_noise = BatchNormalization()(input_noise) # normalized_speech = BatchNormalization()(input_speech) # encoder of net conv_1_1 = Conv2D(filters=32, kernel_size=(1,3),strides=1,padding="valid",kernel_constraint=max_norm(max_norm_value), kernel_initializer="he_normal")(normalized_noisy_1) leakyrelu_1_1 = LeakyReLU()(conv_1_1) conv_1_2 = Conv2D(filters=32, kernel_size=(1,3),strides=1,padding="valid",kernel_constraint=max_norm(max_norm_value), kernel_initializer="he_normal")(leakyrelu_1_1) leakyrelu_1_2 = LeakyReLU()(conv_1_2) conv_1_3 = Conv2D(filters=32, kernel_size=(16,1),strides=1,padding="same", kernel_constraint=max_norm(max_norm_value), kernel_initializer="he_normal")(leakyrelu_1_2) leakyrelu_1_3 = LeakyReLU()(conv_1_3) maxpool_1_1 = MaxPooling2D(pool_size=(2,1))(leakyrelu_1_3) conv_1_4 = Conv2D(filters=32, kernel_size=(16,1),strides=1,padding="same", kernel_constraint=max_norm(max_norm_value), kernel_initializer="he_normal")(maxpool_1_1) leakyrelu_1_4 = LeakyReLU()(conv_1_4) maxpool_1_2 = MaxPooling2D(pool_size=(2,1))(leakyrelu_1_4) conv_1_5 = Conv2D(filters=32, kernel_size=(16,1),strides=1,padding="valid", kernel_constraint=max_norm(max_norm_value), kernel_initializer="he_normal")(maxpool_1_2) leakyrelu_1_5 = LeakyReLU()(conv_1_5) conv_1_6 = Conv2D(filters=32, kernel_size=(16,1),strides=1,padding="valid", kernel_constraint=max_norm(max_norm_value), kernel_initializer="he_normal")(leakyrelu_1_5) leakyrelu_1_6 = LeakyReLU()(conv_1_6) # decoder of Net convtrans_1_1 = Conv2DTranspose(filters=32, kernel_size=(16,1),strides=1,padding="valid", kernel_constraint=max_norm(max_norm_value), kernel_initializer="he_normal")(leakyrelu_1_6) leakyrelu_1_7 = LeakyReLU()(convtrans_1_1) convtrans_1_2 = Conv2DTranspose(filters=32, kernel_size=(16,1),strides=1, padding="valid", kernel_constraint=max_norm(max_norm_value), kernel_initializer="he_normal")(leakyrelu_1_7) leakyrelu_1_8 = LeakyReLU()(convtrans_1_2) skip_1_1 = Add()([maxpool_1_2,leakyrelu_1_8]) up_1_1 = UpSampling2D(size=(2,1))(skip_1_1) conv_1_7 = Conv2D(filters=32, kernel_size=(16,1), strides=1, padding="same", kernel_constraint=max_norm(max_norm_value), kernel_initializer="he_normal")(up_1_1) leakyrelu_1_9 = LeakyReLU()(conv_1_7) skip_1_2 = Add()([leakyrelu_1_4,leakyrelu_1_9]) up_1_2 = UpSampling2D(size=(2,1))(skip_1_2) conv_1_8 = Conv2D(filters=32, kernel_size=(16,1), strides=1, padding="same", kernel_constraint=max_norm(max_norm_value), kernel_initializer="he_normal")(up_1_2) leakyrelu_1_10 = LeakyReLU()(conv_1_8) skip_1_3 = Add()([leakyrelu_1_3,leakyrelu_1_10]) # mask from noisy input mask_1 = Conv2D(filters=1, kernel_size=(16,1),strides=1, padding="same", activation='sigmoid', kernel_constraint=max_norm(max_norm_value), kernel_initializer="he_normal")(skip_1_3) # filtered speech and noise component n_tilde_1 = Multiply()([mask_1,input_noise_1]) s_tilde_1 = Multiply()([mask_1,input_speech_1]) # for secondchannel # defining inputs and normalizing just noisy input input_noisy_2 = Input(shape=(260,5,1)) input_noise_2 = Input(shape=(260,1,1)) input_speech_2 = Input(shape=(260,1,1)) normalized_noisy_2 = BatchNormalization()(input_noisy_2) # normalized_noise = BatchNormalization()(input_noise) # normalized_speech = BatchNormalization()(input_speech) # encoder of net conv_2_1 = Conv2D(filters=32, kernel_size=(1,3),strides=1,padding="valid",kernel_constraint=max_norm(max_norm_value), kernel_initializer="he_normal")(normalized_noisy_2) leakyrelu_2_1 = LeakyReLU()(conv_2_1) conv_2_2 = Conv2D(filters=32, kernel_size=(1,3),strides=1,padding="valid",kernel_constraint=max_norm(max_norm_value), kernel_initializer="he_normal")(leakyrelu_2_1) leakyrelu_2_2 = LeakyReLU()(conv_2_2) conv_2_3 = Conv2D(filters=32, kernel_size=(16,1),strides=1,padding="same", kernel_constraint=max_norm(max_norm_value), kernel_initializer="he_normal")(leakyrelu_2_2) leakyrelu_2_3 = LeakyReLU()(conv_2_3) maxpool_2_1 = MaxPooling2D(pool_size=(2,1))(leakyrelu_2_3) conv_2_4 = Conv2D(filters=32, kernel_size=(16,1),strides=1,padding="same", kernel_constraint=max_norm(max_norm_value), kernel_initializer="he_normal")(maxpool_2_1) leakyrelu_2_4 = LeakyReLU()(conv_2_4) maxpool_2_2 = MaxPooling2D(pool_size=(2,1))(leakyrelu_2_4) conv_2_5 = Conv2D(filters=32, kernel_size=(16,1),strides=1,padding="valid", kernel_constraint=max_norm(max_norm_value), kernel_initializer="he_normal")(maxpool_2_2) leakyrelu_2_5 = LeakyReLU()(conv_2_5) conv_2_6 = Conv2D(filters=32, kernel_size=(16,1),strides=1,padding="valid", kernel_constraint=max_norm(max_norm_value), kernel_initializer="he_normal")(leakyrelu_2_5) leakyrelu_2_6 = LeakyReLU()(conv_2_6) # decoder of Net convtrans_2_1 = Conv2DTranspose(filters=32, kernel_size=(16,1),strides=1,padding="valid", kernel_constraint=max_norm(max_norm_value), kernel_initializer="he_normal")(leakyrelu_2_6) leakyrelu_2_7 = LeakyReLU()(convtrans_2_1) convtrans_2_2 = Conv2DTranspose(filters=32, kernel_size=(16,1),strides=1, padding="valid", kernel_constraint=max_norm(max_norm_value), kernel_initializer="he_normal")(leakyrelu_2_7) leakyrelu_2_8 = LeakyReLU()(convtrans_2_2) skip_2_1 = Add()([maxpool_2_2,leakyrelu_2_8]) up_2_1 = UpSampling2D(size=(2,1))(skip_2_1) conv_2_7 = Conv2D(filters=32, kernel_size=(16,1), strides=1, padding="same", kernel_constraint=max_norm(max_norm_value), kernel_initializer="he_normal")(up_2_1) leakyrelu_2_9 = LeakyReLU()(conv_2_7) skip_2_2 = Add()([leakyrelu_2_4,leakyrelu_2_9]) up_2_2 = UpSampling2D(size=(2,1))(skip_2_2) conv_2_8 = Conv2D(filters=32, kernel_size=(16,1), strides=1, padding="same", kernel_constraint=max_norm(max_norm_value), kernel_initializer="he_normal")(up_2_2) leakyrelu_2_10 = LeakyReLU()(conv_2_8) skip_2_3 = Add()([leakyrelu_2_3,leakyrelu_2_10]) # mask from noisy input mask_2 = Conv2D(filters=1, kernel_size=(16,1),strides=1, padding="same", activation='sigmoid', kernel_constraint=max_norm(max_norm_value), kernel_initializer="he_normal")(skip_2_3) # filtered speech and noise component n_tilde_2 = Multiply()([mask_2,input_noise_2]) s_tilde_2 = Multiply()([mask_2,input_speech_2]) # defining model model = Model(inputs=[input_noisy_1,input_noise_1,input_speech_1,input_noisy_2,input_noise_2,input_speech_2], outputs=[n_tilde_1,s_tilde_1,n_tilde_2,s_tilde_2]) return model # if __name__ == "__main__": # build model and compile it model = build_model() model.compile(loss='mse', loss_weights=[0.25, 0.25, 0.25, 0.25], optimizer='adam', metrics=['mape','accuracy']) # making directory for weights if not os.path.exists(path_weights_model): try: os.makedirs(path_weights_model) except Exception as e: print("Konnte Ordner fuer Gewichte nicht erstellen" + str(e)) # defining how weights are saved filepath = path_weights_model + "/weights-{epoch:04d}.hdf5" checkpoint = ModelCheckpoint( filepath, monitor='loss', verbose=0, save_best_only=True, mode='min' ) model.summary() # train model history = model.fit(train_generator, steps_per_epoch = int(np.floor(len_train // reduction_divisor / batch_size)), epochs = epochs, validation_data = valid_generator, validation_steps = int(np.floor(len_valid // reduction_divisor / batch_size)), callbacks=[checkpoint], use_multiprocessing=True) model.summary() # save model after training model.save(path_save_model) index = path_save_model.split("/") path_save_nomodel = '/'.join(index[0:-1]) loss = history.history["loss"] multiply_1_loss = history.history["multiply_1_loss"] multiply_2_loss = history.history["multiply_2_loss"] multiply_3_loss = history.history["multiply_3_loss"] multiply_4_loss = history.history["multiply_4_loss"] multiply_1_mape = history.history["multiply_1_mape"] multiply_2_mape = history.history["multiply_2_mape"] multiply_3_mape = history.history["multiply_3_mape"] multiply_4_mape = history.history["multiply_4_mape"] multiply_1_accuracy = history.history["multiply_1_accuracy"] multiply_2_accuracy = history.history["multiply_2_accuracy"] multiply_3_accuracy = history.history["multiply_3_accuracy"] multiply_4_accuracy = history.history["multiply_4_accuracy"] val_loss = history.history["val_loss"] val_multiply_1_loss = history.history["val_multiply_1_loss"] val_multiply_2_loss = history.history["val_multiply_2_loss"] val_multiply_3_loss = history.history["val_multiply_3_loss"] val_multiply_4_loss = history.history["val_multiply_4_loss"] val_multiply_1_mape = history.history["val_multiply_1_mape"] val_multiply_2_mape = history.history["val_multiply_2_mape"] val_multiply_3_mape = history.history["val_multiply_3_mape"] val_multiply_4_mape = history.history["val_multiply_4_mape"] val_multiply_1_accuracy = history.history["val_multiply_1_accuracy"] val_multiply_2_accuracy = history.history["val_multiply_2_accuracy"] val_multiply_3_accuracy = history.history["val_multiply_3_accuracy"] val_multiply_4_accuracy = history.history["val_multiply_4_accuracy"] dict_history = {'loss':loss, 'multiply_1_loss':multiply_1_loss, 'multiply_2_loss':multiply_2_loss, 'multiply_3_loss':multiply_3_loss, 'multiply_4_loss':multiply_4_loss, 'multiply_1_mape':multiply_1_mape, 'multiply_2_mape':multiply_2_mape, 'multiply_3_mape':multiply_3_mape, 'multiply_4_mape':multiply_4_mape, 'multiply_1_accuracy':multiply_1_accuracy, 'multiply_2_accuracy':multiply_2_accuracy, 'multiply_3_accuracy':multiply_3_accuracy, 'multiply_4_accuracy':multiply_4_accuracy, 'val_loss':val_loss, 'val_multiply_1_loss':val_multiply_1_loss, 'val_multiply_2_loss':val_multiply_2_loss, 'val_multiply_3_loss':val_multiply_3_loss, 'val_multiply_4_loss':val_multiply_4_loss, 'val_multiply_1_mape':val_multiply_1_mape, 'val_multiply_2_mape':val_multiply_2_mape, 'val_multiply_3_mape':val_multiply_3_mape, 'val_multiply_4_mape':val_multiply_4_mape, 'val_multiply_1_accuracy':val_multiply_1_accuracy, 'val_multiply_2_accuracy':val_multiply_2_accuracy, 'val_multiply_3_accuracy':val_multiply_3_accuracy, 'val_multiply_4_accuracy':val_multiply_4_accuracy} # loss_history = history.history["loss"] # mape_history = history.history["mape"] # accuracy_history = history.history["accuracy"] # val_loss_history = history.history["val_loss"] # val_mape_history = history.history["val_mape"] # val_accuracy_history = history.history["val_accuracy"] # # dict_history = {'loss':loss_history, 'mape':mape_history, 'accuracy':accuracy_history, 'val_loss':val_loss_history, 'val_mape':val_mape_history, 'val_accuracy':val_accuracy_history} np.save(path_save_nomodel + '/' + plot_name, dict_history) # plot train loss and validation loss after training plt.style.use("ggplot") plt.figure() plt.plot(history.history["loss"], label="train_loss") plt.plot(history.history["val_loss"], label="val_loss") plt.title("Training Loss and Accuracy") plt.xlabel("Epochs") plt.ylabel("Loss/Accuracy") plt.legend(loc="lower left") index = path_save_model.split("/") path_save_nomodel = '/'.join(index[0:-1]) plt.savefig(path_save_nomodel + '/' + plot_name + '.png', dpi=400) # plt.show() return