Keras CNN Sound Classify #3

1. 学習の実行
プログラムは、かきになります。
tarin-mobile.py
''' sound classify train-mobile.py from keras.applications import MobileNetV2 https://qiita.com/cvusk/items/61cdbce80785eaf28349 ''' import os import random import numpy as np #import pandas as pd #import tensorflow as tf #import tensorflow.keras.backend as K import keras #from keras.applications import VGG16 #from keras.applications import MobileNetV2 from keras.applications import MobileNet from keras.models import Model from keras.layers import Input, Dense, Dropout, Activation from keras.layers import Conv2D, GlobalAveragePooling2D from keras.layers import BatchNormalization, Add from keras.callbacks import TensorBoard, EarlyStopping, ModelCheckpoint from keras.optimizers import Adam import matplotlib.pyplot as plt #config = tf.ConfigProto() #config.gpu_options.allow_growth = True #K.set_session(tf.Session(config=config)) # https://www.haya-programming.com/entry/2018/12/28/004211 npz_list = ['esc_melsp_test.npz', 'esc_melsp_train_raw.npz', 'esc_melsp_train_wn.npz', 'esc_melsp_train_ss.npz', 'esc_melsp_train_st.npz', 'esc_melsp_train_com.npz' ] classes = 50 test_id=3 log_dir = 'logs/mobile-test3/' class MixupGenerator(): def __init__(self, X_train, y_train, batch_size=32, alpha=0.2, shuffle=True, datagen=None): self.X_train = X_train self.y_train = y_train self.batch_size = batch_size self.alpha = alpha self.shuffle = shuffle self.sample_num = len(X_train) self.datagen = datagen def __call__(self): while True: indexes = self.__get_exploration_order() itr_num = int(len(indexes) // (self.batch_size * 2)) for i in range(itr_num): batch_ids = indexes[i * self.batch_size * 2:(i + 1) * self.batch_size * 2] X, y = self.__data_generation(batch_ids) yield X, y def __get_exploration_order(self): indexes = np.arange(self.sample_num) if self.shuffle: np.random.shuffle(indexes) return indexes def __data_generation(self, batch_ids): _, h, w, c = self.X_train.shape _, class_num = self.y_train.shape X1 = self.X_train[batch_ids[:self.batch_size]] X2 = self.X_train[batch_ids[self.batch_size:]] y1 = self.y_train[batch_ids[:self.batch_size]] y2 = self.y_train[batch_ids[self.batch_size:]] l = np.random.beta(self.alpha, self.alpha, self.batch_size) X_l = l.reshape(self.batch_size, 1, 1, 1) y_l = l.reshape(self.batch_size, 1) X = X1 * X_l + X2 * (1 - X_l) y = y1 * y_l + y2 * (1 - y_l) if self.datagen: for i in range(self.batch_size): X[i] = self.datagen.random_transform(X[i]) return X, y def load_npz(path,classes=50, test=3): npz_dt= np.load(path) #print('list(npz_dt)=',list(npz_dt)) x = npz_dt['x'].astype('float32') # data type convert float63 -> float32 #x = x.astype('float32') #x=x[:,:,:344] x_shape = x.shape #print(x.shape) #sys.exit() # reshape # (-1,128,1723) -> (-1,128,1723,1) x = np.reshape(x, (-1,x_shape[1],x_shape[2],1)) # normalize if test == 2: x_max=x.max() x_min=x.min() if x_min < 0.0 : x_min *= -1.0 x += x_min x_max += x_min #x /= x_max x /= 6.5 #if x_max > 4.0: print('>x_max=',x_max) if test ==3: x /= 4.1 x_max=x.max() x_min=x.min() print('>x_max=',x_max) print('>x_min=',x_min) if test ==4: x_max=x.max() x_min=x.min() print('>x_max=',x_max) print('>x_min=',x_min) y = npz_dt['y'] # redefine target data into one hot vector y = keras.utils.to_categorical(y, classes) return x ,y # load test data x_test,y_test = load_npz(npz_list[0],classes=classes,test=test_id) # define CNN #print('x_train.shape=',x_train.shape) #x_train.shape= (1500, 128, 1723, 1) print('>x_test.shape=',x_test.shape) print('>x_test.dtype=',x_test.dtype) model =MobileNet(weights=None,input_shape=(128,1723,1),classes=classes,dropout=0.4) #model = MobileNetV2(weights=None,alpha=0.5,input_shape=(128,1723,1),classes=classes) #logging = TensorBoard(log_dir=log_dir,write_graph=True) logging = TensorBoard(log_dir=log_dir) next_epochs=75 epochs=5 batch_size=6 #batch_size=4 weights_load_path = 'sound_trained_weights-mobile-test3-5_3.h5' weights_sav_path = 'sound_trained_weights-mobile-test3-6' if True: model.load_weights(weights_load_path, by_name=True) if next_epochs >= 45: # initiate Adam optimizer #opt = Adam(lr=0.00001, decay=1e-6, amsgrad=True) opt=Adam(lr=1e-4) # 0.001 else: opt=Adam(lr=1e-3) # 0.01 # Let's train the model using Adam with amsgrad model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy']) #model.summary() #for id in range(1,6): for id in range(1,4): print('npz=',npz_list[id]) # load train data x_train,y_train = load_npz(npz_list[id],classes=classes,test=test_id) if False: hist=model.fit(x_train,y_train, #epochs=50, epochs=next_epochs+epochs, initial_epoch=next_epochs, batch_size=batch_size, validation_data=(x_test, y_test), callbacks=[logging]) if True: training_generator = MixupGenerator(x_train, y_train,batch_size=batch_size)() hist=model.fit_generator(generator=training_generator, steps_per_epoch=x_train.shape[0] // batch_size, validation_data=(x_test, y_test), epochs=next_epochs+epochs, initial_epoch=next_epochs, # verbose=1, callbacks=[logging]) sav_p= "%s_%d.h5" % (weights_sav_path, id) print('sav_p=',sav_p) model.save_weights(sav_p) #------------- if False: hist_dic=hist.history #print(hist_dic.keys()) #dict_keys(['val_loss', 'val_accuracy', 'loss', 'accuracy'] acc = hist_dic['accuracy'] val_acc = hist_dic['val_accuracy'] loss = hist_dic['loss'] val_loss = hist_dic['val_loss'] epochs = range(1, len(acc) + 1) # "bo" is for "blue dot" plt.plot(epochs, loss, 'bo', label='Training loss') # b is for "solid blue line" plt.plot(epochs, val_loss, 'b', label='Validation loss') plt.title('Training and validation loss') plt.xlabel('Epochs') plt.ylabel('Loss') plt.legend() plt.show() plt.clf() plt.plot(epochs, acc, 'bo', label='Training acc') # b is for "solid blue line" plt.plot(epochs, val_acc, 'b', label='Validation acc') plt.title('Training and validation accuracy') plt.xlabel('Epochs') plt.ylabel('Accuracy') plt.legend() plt.show() x_train=None y_train=None next_epochs+=epochs #model.save_weights(weights_sav_path)

2. 学習の結果
90 epoch まで学習させましたが、
1 epoch 120[sec] なので、直ぐに終わります。
tennsorboard で確認すると、50 epoch でMAX みたいです。
Epoch 46/50 250/250 [==============================] - 120s 482ms/step - loss: 0.7923 - accuracy: 0.9433 - val_loss: 0.8175 - val_accuracy: 0.8000 Epoch 47/50 250/250 [==============================] - 99s 394ms/step - loss: 0.7577 - accuracy: 0.9460 - val_loss: 0.7608 - val_accuracy: 0.8140 Epoch 48/50 250/250 [==============================] - 98s 393ms/step - loss: 0.7798 - accuracy: 0.9393 - val_loss: 0.7438 - val_accuracy: 0.8040 Epoch 49/50 250/250 [==============================] - 99s 395ms/step - loss: 0.7597 - accuracy: 0.9407 - val_loss: 0.7458 - val_accuracy: 0.8100 Epoch 50/50 250/250 [==============================] - 99s 394ms/step - loss: 0.6985 - accuracy: 0.9507 - val_loss: 0.7512 - val_accuracy: 0.8040 sav_p= sound_trained_weights-mobile-test3-4_1.h5 npz= esc_melsp_train_wn.npz >x_max= 0.5990754 >x_min= -0.75613326 Epoch 51/55 250/250 [==============================] - 100s 398ms/step - loss: 0.7612 - accuracy: 0.9440 - val_loss: 0.7662 - val_accuracy: 0.7980 Epoch 52/55 250/250 [==============================] - 99s 397ms/step - loss: 0.7397 - accuracy: 0.9360 - val_loss: 0.7457 - val_accuracy: 0.8020 Epoch 53/55 250/250 [==============================] - 99s 395ms/step - loss: 0.7347 - accuracy: 0.9407 - val_loss: 0.7576 - val_accuracy: 0.8120 Epoch 54/55 250/250 [==============================] - 99s 395ms/step - loss: 0.7400 - accuracy: 0.9347 - val_loss: 0.7603 - val_accuracy: 0.8140 Epoch 55/55 250/250 [==============================] - 99s 396ms/step - loss: 0.7075 - accuracy: 0.9473 - val_loss: 0.8132 - val_accuracy: 0.8180 sav_p= sound_trained_weights-mobile-test3-4_2.h5 npz= esc_melsp_train_ss.npz >x_max= 0.59915954 >x_min= -0.9708573 Epoch 86/90 250/250 [==============================] - 102s 408ms/step - loss: 0.6395 - accuracy: 0.9527 - val_loss: 0.7430 - val_accuracy: 0.8180 Epoch 87/90 250/250 [==============================] - 99s 395ms/step - loss: 0.6451 - accuracy: 0.9573 - val_loss: 0.8014 - val_accuracy: 0.8120 Epoch 88/90 250/250 [==============================] - 98s 393ms/step - loss: 0.6642 - accuracy: 0.9527 - val_loss: 0.7854 - val_accuracy: 0.8160 Epoch 89/90 250/250 [==============================] - 99s 395ms/step - loss: 0.6483 - accuracy: 0.9573 - val_loss: 0.7585 - val_accuracy: 0.8200 Epoch 90/90 250/250 [==============================] - 99s 395ms/step - loss: 0.6317 - accuracy: 0.9613 - val_loss: 0.7623 - val_accuracy: 0.8180 sav_p= sound_trained_weights-mobile-test3-6_3.h5

3. 検証プログラム
ESC-50 のmeta/esc50.csv　を読み込んで、
実際の音を、ヘッドホーンで確認しながら確認してみました。

プログラムは、下記になります。
sound-predict-mobile.py
''' Created on 2020/01/27 sound-predict-mobile.py https://qiita.com/__Attsun__/items/e033d689c336315435b3 https://musicinformationretrieval.com/ipython_audio.html @author: nishi ''' import keras from keras.models import Model from keras.applications import MobileNet from keras.layers import Input, Dense, Dropout, Activation from keras.layers import Conv2D, GlobalAveragePooling2D from keras.layers import BatchNormalization, Add from keras.callbacks import TensorBoard, EarlyStopping, ModelCheckpoint import matplotlib.pyplot as plt import librosa import librosa.display #import matplotlib.pyplot as plt import numpy as np import pandas as pd import matplotlib as mpl import pyaudio import os import glob #base_dir='F:/tmp/ESC-50/ESC-50-master/audio/' base_dir='F:/tmp/ESC-50/' esc_dir = os.path.join(base_dir, "ESC-50-master") meta_file = os.path.join(esc_dir, "meta/esc50.csv") audio_dir = os.path.join(esc_dir, "audio/") test_id=3 check_id=3 class_indx={ 0: 'dog', 1: 'rooster', 2: 'pig', 3: 'cow', 4: 'frog', 5: 'cat', 6: 'hen', 7: 'insects', 8: 'sheep', 9: 'crow', 10: 'rain', 11: 'sea_waves', 12: 'crackling_fire', 13: 'crickets', 14: 'chirping_birds', 15: 'water_drops', 16: 'wind', 17: 'pouring_water', 18: 'toilet_flush', 18: 'toilet_flush', 19: 'thunderstorm', 20: 'crying_baby', 21: 'sneezing', 22: 'clapping', 23: 'breathing', 24: 'coughing', 25: 'footsteps', 26: 'laughing', 27: 'brushing_teeth', 28: 'snoring', 29: 'drinking_sipping', 30: 'door_wood_knock', 31: 'mouse_click', 32: 'keyboard_typing', 33: 'door_wood_creaks', 34: 'can_opening', 35: 'washing_machine', 36: 'vacuum_cleaner', 37: 'clock_alarm', 38: 'clock_tick', 39: 'glass_breaking', 40: 'helicopter', 41: 'chainsaw', 42: 'siren', 43: 'car_horn', 44: 'engine', 45: 'train', 46: 'church_bells', 47: 'airplane', 48: 'fireworks', 49: 'hand_saw' } if check_id==1: f_list=['1-137-A-32.wav','1-13572-A-46.wav','1-50455-A-44.wav','1-208757-D-2.wav','4-182041-A-30.wav'] if check_id==2: f_list2 = glob.glob(audio_dir+'*') if check_id==3: # load metadata meta_data = pd.read_csv(meta_file) # get data size data_size = meta_data.shape print(data_size) #print(meta_data.keys()) #Index(['filename', 'fold', 'target', 'category', 'esc10', 'src_file', 'take'], dtype='object') print(meta_data.loc[1,"filename"]) #class_dict = {} #for i in range(data_size[0]): # if meta_data.loc[i,"target"] not in class_dict.keys(): # class_dict[meta_data.loc[i,"target"]] = meta_data.loc[i,"category"] #print(class_dict) # load a wave data def load_wave_data(audio_dir, file_name): file_path = os.path.join(audio_dir, file_name) x, fs = librosa.load(file_path, sr=44100) return x,fs # change wave data to mel-stft def calculate_melsp(x, n_fft=1024, hop_length=128): stft = np.abs(librosa.stft(x, n_fft=n_fft, hop_length=hop_length))**2 log_stft = librosa.power_to_db(stft) melsp = librosa.feature.melspectrogram(S=log_stft,n_mels=128) return melsp def load_melsp(dx,test=2): dx=dx[np.newaxis,:,:] #print(dx.shape) #print(dx.dtype) #x = x.astype('float32') # data type convert float63 -> float32 #x = x.astype('float32') dx_shape = dx.shape # reshape # (-1,128,1723) -> (-1,128,1723,1) dx = np.reshape(dx, (-1,dx_shape[1],dx_shape[2],1)) #print(dx.shape) # normalize if test == 2: dx_max=dx.max() dx_min=dx.min() print('>dx_max=',dx_max,', dx_min=',dx_min) if dx_min < 0.0 : dx_min *= -1.0 dx += dx_min dx_max += dx_min #dx /= dx_max dx /= 6.5 #if dx_max > 4.0: print('>dx_max=',dx_max) if test==3: dx /= 4.1 dx_max=dx.max() dx_min=dx.min() print('>dx_max=',dx_max) print('>dx_min=',dx_min) if test ==4: dx_max=dx.max() dx_min=dx.min() print('>dx_max=',dx_max) print('>dx_min=',dx_min) return dx classes = 50 model =MobileNet(weights=None,input_shape=(128,1723,1),classes=classes,dropout=0.4) weights_load_path = 'save/sound_trained_weights-mobile-test3-6_3.h5' model.load_weights(weights_load_path, by_name=True) #CHUNK=1024 CHUNK=2**11 RATE=44100 p=pyaudio.PyAudio() stream=p.open(format = pyaudio.paInt16, channels = 1, rate = RATE, frames_per_buffer = CHUNK, input = True, output = True) # input縺ｨoutput繧貞酔譎ゅ↓True縺ｫ縺吶ｋ while stream.is_active(): #input = stream.read(CHUNK) #output = stream.write(input) #for f in f_list: # dx, sr = librosa.load(audio_dir+f,sr=44100) #for f in f_list2: # dx, sr = librosa.load(f,sr=44100) # #dx, sr = load_wave_data(audio_dir, f) core_cnt=0; for h in range(data_size[0]): dx, sr = load_wave_data(audio_dir, meta_data.loc[h,"filename"]) act_category = meta_data.loc[h,"category"] act_target = meta_data.loc[h,"target"] #print(dx.shape) #(220500,) #print(dx.dtype) #print(sr) dx_max=dx.max() dx_min=dx.min() print('dx_max=',dx_max) print('dx_min=',dx_min) #j=dx*256.0*25.0 j=dx*2**15 j=j.astype('int16') output = stream.write(j) melsp = calculate_melsp(dx) dt=load_melsp(melsp,test=test_id) v = model.predict(dt) i=v.argmax() s=class_indx[i] if i == act_target: core_cnt += 1 acc = float(core_cnt/(h+1)) ss="%d/%d acc=%f" % (core_cnt,h+1,acc) print('>',i,':',s,' ',v.max(),',act=',act_category,ss) if False: #plt.figure(figsize=(14, 5)) #plt.figure(figsize=(11, 5)) librosa.display.waveplot(dx, sr=sr) plt.show() if False: X = librosa.stft(dx) Xdb = librosa.amplitude_to_db(abs(X)) plt.figure(figsize=(14, 5)) librosa.display.specshow(Xdb, sr=sr, x_axis='time', y_axis='hz') plt.show() break stream.stop_stream() stream.close() p.terminate()

4. 検証の結果
最後の、２件の結果だけ示します。
学習とテストデータがごちゃ混ぜなので、結果として
学習時の正解率となりました。
正解率: 0.95

dx_max= 0.821228 dx_min= -1.0 >dx_max= 0.3883457 >dx_min= -0.48607498 > 8 : sheep 0.18795581 ,act= sheep 1908/1999 acc=0.954477 dx_max= 0.84399414 dx_min= -0.95739746 >dx_max= 0.48210397 >dx_min= -0.48193836 > 0 : dog 0.9987759 ,act= dog 1909/2000 acc=0.954500

5.結論
ESC-50 の WAV はS/N 比ががよくて、クリアーな音で、旨く録音されているので
結構、良い結果が得られたと思います。

これが、PCのマイクからだと、かなり音が悪いので、同じ様な結果は得られません。
PCのマイク音のノイズやダイナミックレンジを改善するか、
PCのマイク音に近い音質の学習データも加えてやれれば、かなり良くなるのではと思います。

注) この点に関しては、Keras CNN Sound Classify #2 で ESC-50 の WAV音を、スピーカー出力 > マイク入力
での評価テストの中で、高音域の消失(減衰) が問題だと言う事判ったので、そちらを参考にしてください。
つまり、アプリケーションプログラムの実装の問題。

この後、MobileNet V2 も同じように試してみましたが、
こちらは、GTX1050 の 4GB では、役不足です。
8GBの GPU が買えたら、再チャレンジしてみたいと思もっちょります。
注) float16 を使うと、GTX1050 でも学習できるようなので、今試しちょります。(2010.2.7 by おんちゃん)

6.その後
1) コンデンサーマイクが、音がクリアーだと言う事なので、早速、安いコンデンサーマイクを
購入して試してみます。

2) 当初、K210(Sipeed MAIX BiT) で、このMobileNet 版 sound-classify を動かそうかと思っていましたが、
その前に、Coral USB TPU Accelerator(EdgeTPU) + Raspberry PI 3B+ が簡単に
出来そうなので、Ubuntu PC + Coral USB TPU Accelerator(EdgeTPU) で先に試して見ようか
と考えちょります。