Coral Edge TPU USB のサンプルの Object Detection を試す。#2

1. wav ファイルの作成 。

取り敢えず、下記プログラムで、"コップ　ぞね","ねこ　ぞね","犬　ぞね" と発声する wav ファイルを
それぞれ、ファイル名称を変えて作成します。
neko.wav , inu.wav , kop.wav

sayaka-wav.py
# -*- coding:utf-8 -*- ''' sayaka-wav.py https://qiita.com/7shi/items/a5fb03406e0626b4f138 Created on 2020/07/21 @author: nishi ''' import win32com.client sapi = win32com.client.Dispatch("SAPI.SpVoice") cat = win32com.client.Dispatch("SAPI.SpObjectTokenCategory") cat.SetID(r"HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Speech_OneCore\Voices", False) v = [t for t in cat.EnumerateTokens() if t.GetAttribute("Name") == "Microsoft Sayaka"] if v: fs = win32com.client.Dispatch("SAPI.SpFileStream") fs.Open("sayaka.wav", 3) sapi.AudioOutputStream = fs oldv = sapi.Voice sapi.Voice = v[0] #sapi.Speak("こんにちは、世界") #sapi.Speak("コップ　ぞね") #sapi.Speak("ねこ　ぞね") sapi.Speak("犬　ぞね") sapi.Voice = oldv fs.Close()

2. 次は、コップを検出したら、"コップぞね" と発声するプログラムの作成です。

detect-speaker.py
# -*- coding:utf-8 -*- # Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """A demo that runs object detection on camera frames using OpenCV. TEST_DATA=../all_models Run coco model: python3 detect-speaker.py \ --model ${TEST_DATA}/mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite \ --labels ${TEST_DATA}/coco_labels.txt """ import argparse import collections import common import cv2 import numpy as np import os from PIL import Image import re import tflite_runtime.interpreter as tflite # add by nishi from timeit import default_timer as timer import librosa import pyaudio Object = collections.namedtuple('Object', ['id', 'score', 'bbox']) def load_labels(path): p = re.compile(r'\s*(\d+)(.+)') with open(path, 'r', encoding='utf-8') as f: lines = (p.match(line).groups() for line in f.readlines()) return {int(num): text.strip() for num, text in lines} class BBox(collections.namedtuple('BBox', ['xmin', 'ymin', 'xmax', 'ymax'])): """Bounding box. Represents a rectangle which sides are either vertical or horizontal, parallel to the x or y axis. """ __slots__ = () def get_output(interpreter, score_threshold, top_k, image_scale=1.0): """Returns list of detected objects.""" boxes = common.output_tensor(interpreter, 0) class_ids = common.output_tensor(interpreter, 1) scores = common.output_tensor(interpreter, 2) count = int(common.output_tensor(interpreter, 3)) def make(i): ymin, xmin, ymax, xmax = boxes[i] return Object( id=int(class_ids[i]), score=scores[i], bbox=BBox(xmin=np.maximum(0.0, xmin), ymin=np.maximum(0.0, ymin), xmax=np.minimum(1.0, xmax), ymax=np.minimum(1.0, ymax))) return [make(i) for i in range(top_k) if scores[i] >= score_threshold] # load a wave data def load_wave_data(audio_dir, file_name): file_path = os.path.join(audio_dir, file_name) x, fs = librosa.load(file_path, sr=44100) return x,fs def load_wave_tb(): wav_tb={} dx, sr = load_wave_data('.', 'neko.wav') j=dx*2**15 j=j.astype('int16') wav_tb[16]=j # cat=16 #output = stream.write(wav_tb[16],wav_tb[16].shape[0]) dx, sr = load_wave_data('.', 'inu.wav') j=dx*2**15 j=j.astype('int16') wav_tb[17]=j # dog=17 #output = stream.write(wav_tb[17],wav_tb[17].shape[0]) dx, sr = load_wave_data('.', 'kop.wav') j=dx*2**15 j=j.astype('int16') wav_tb[46]=j # cup=46 #output = stream.write(wav_tb[46],wav_tb[46].shape[0]) return wav_tb def main(): default_model_dir = '../all_models' default_model = 'mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite' default_labels = 'coco_labels.txt' parser = argparse.ArgumentParser() parser.add_argument('--model', help='.tflite model path', default=os.path.join(default_model_dir,default_model)) parser.add_argument('--labels', help='label file path', default=os.path.join(default_model_dir, default_labels)) parser.add_argument('--top_k', type=int, default=3, help='number of categories with highest score to display') parser.add_argument('--camera_idx', type=int, help='Index of which video source to use. ', default = 0) parser.add_argument('--threshold', type=float, default=0.5, help='classifier score threshold') args = parser.parse_args() #CHUNK=1024 CHUNK=2**11 RATE=44100 p=pyaudio.PyAudio() stream=p.open(format = pyaudio.paInt16, channels = 1, rate = RATE, frames_per_buffer = CHUNK, input = False, output = True) # input��output�𓯎���True�ɂ��� wav_tb=load_wave_tb() #output = stream.write(wav_tb[16],wav_tb[16].shape[0]) #output = stream.write(wav_tb[17],wav_tb[17].shape[0]) #output = stream.write(wav_tb[46],wav_tb[46].shape[0]) print('Loading {} with {} labels.'.format(args.model, args.labels)) interpreter = common.make_interpreter(args.model) interpreter.allocate_tensors() if False: input_fs = interpreter.get_input_details() output_fs = interpreter.get_output_details() print('>input_fs=',input_fs) print('>output_fs=',output_fs) labels = load_labels(args.labels) cap = cv2.VideoCapture(args.camera_idx) # test by nishi for fps accum_time = 0 curr_fps = 0 fps = "FPS: ??" prev_time = timer() while cap.isOpened(): ret, frame = cap.read() if not ret: break cv2_im = frame cv2_im_rgb = cv2.cvtColor(cv2_im, cv2.COLOR_BGR2RGB) pil_im = Image.fromarray(cv2_im_rgb) common.set_input(interpreter, pil_im) interpreter.invoke() objs = get_output(interpreter, score_threshold=args.threshold, top_k=args.top_k) cv2_im = append_objs_to_img(cv2_im, objs, labels) # add by nishi for fps start curr_time = timer() exec_time = curr_time - prev_time prev_time = curr_time accum_time = accum_time + exec_time curr_fps = curr_fps + 1 if accum_time > 1: accum_time = accum_time - 1 fps = "FPS: " + str(curr_fps) curr_fps = 0 cv2.putText(cv2_im, text=fps, org=(3, 15), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=0.50, color=(255, 0, 0), thickness=2) #print('>type(cv2_im)=',type(cv2_im)) #print('>cv2_im.shape=',cv2_im.shape) #print('>cv2_im.dtype=',cv2_im.dtype) cv2.imshow('frame', cv2_im) if cv2.waitKey(1) & 0xFF == ord('q'): break obj_speaker(stream,objs,wav_tb) cap.release() cv2.destroyAllWindows() def append_objs_to_img(cv2_im, objs, labels): height, width, channels = cv2_im.shape for obj in objs: x0, y0, x1, y1 = list(obj.bbox) x0, y0, x1, y1 = int(x0*width), int(y0*height), int(x1*width), int(y1*height) percent = int(100 * obj.score) label = '{}% {}'.format(percent, labels.get(obj.id, obj.id)) cv2_im = cv2.rectangle(cv2_im, (x0, y0), (x1, y1), (0, 255, 0), 2) cv2_im = cv2.putText(cv2_im, label, (x0, y0+30), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (255, 0, 0), 2) return cv2_im def obj_speaker(stream,objs,wav_tb): for obj in objs: if obj.id in wav_tb: output = stream.write(wav_tb[obj.id],wav_tb[obj.id].shape[0]) if __name__ == '__main__': main()

>python detect-speaker.py
で実行します。

上記、プログラムで、猫、犬、カップ を検出すると、それぞれ、
"ねこ　ぞね"、"犬　ぞね"、"コップ　ぞね"  と発声します。
まあ、発声している時は、映像は止まりますが、これも、お愛嬌のうちぞね。

この後は、 Raspberry Pi 3B+  と Coral Edge TPU USB で、猫を検出したら、"ねこ ぞね"
と言わせてみようかと、考えちょります。

3. Raspberry Pi 3B+ で走らせてみました。

Raspberry Pi 3B+ and Buster

$python detect-speaker.py Expression 'alsa_snd_pcm_hw_params_set_period_size_near( pcm, hwParams, &alsaPeriodFrames, &dir )' failed in 'src/hostapi/alsa/pa_linux_alsa.c', line: 924 Expression 'alsa_snd_pcm_hw_params_set_period_size_near( pcm, hwParams, &alsaPeriodFrames, &dir )' failed in 'src/hostapi/alsa/pa_linux_alsa.c', line: 924 Expression 'alsa_snd_pcm_hw_params_set_period_size_near( pcm, hwParams, &alsaPeriodFrames, &dir )' failed in 'src/hostapi/alsa/pa_linux_alsa.c', line: 924 ALSA lib pcm.c:2565:(snd_pcm_open_noupdate) Unknown PCM cards.pcm.front ALSA lib pcm.c:2565:(snd_pcm_open_noupdate) Unknown PCM cards.pcm.rear ALSA lib pcm.c:2565:(snd_pcm_open_noupdate) Unknown PCM cards.pcm.center_lfe ALSA lib pcm.c:2565:(snd_pcm_open_noupdate) Unknown PCM cards.pcm.side ALSA lib pcm.c:2565:(snd_pcm_open_noupdate) Unknown PCM cards.pcm.surround21 ALSA lib pcm.c:2565:(snd_pcm_open_noupdate) Unknown PCM cards.pcm.surround21 ALSA lib pcm.c:2565:(snd_pcm_open_noupdate) Unknown PCM cards.pcm.surround40 ALSA lib pcm.c:2565:(snd_pcm_open_noupdate) Unknown PCM cards.pcm.surround41 ALSA lib pcm.c:2565:(snd_pcm_open_noupdate) Unknown PCM cards.pcm.surround50 ALSA lib pcm.c:2565:(snd_pcm_open_noupdate) Unknown PCM cards.pcm.surround51 ALSA lib pcm.c:2565:(snd_pcm_open_noupdate) Unknown PCM cards.pcm.surround71 ALSA lib pcm.c:2565:(snd_pcm_open_noupdate) Unknown PCM cards.pcm.iec958 ALSA lib pcm.c:2565:(snd_pcm_open_noupdate) Unknown PCM cards.pcm.iec958 ALSA lib pcm.c:2565:(snd_pcm_open_noupdate) Unknown PCM cards.pcm.iec958 ALSA lib pcm.c:2565:(snd_pcm_open_noupdate) Unknown PCM cards.pcm.hdmi ALSA lib pcm.c:2565:(snd_pcm_open_noupdate) Unknown PCM cards.pcm.hdmi ALSA lib pcm.c:2565:(snd_pcm_open_noupdate) Unknown PCM cards.pcm.modem ALSA lib pcm.c:2565:(snd_pcm_open_noupdate) Unknown PCM cards.pcm.modem ALSA lib pcm.c:2565:(snd_pcm_open_noupdate) Unknown PCM cards.pcm.phoneline ALSA lib pcm.c:2565:(snd_pcm_open_noupdate) Unknown PCM cards.pcm.phoneline ALSA lib confmisc.c:1281:(snd_func_refer) Unable to find definition 'defaults.bluealsa.device' ALSA lib conf.c:4568:(_snd_config_evaluate) function snd_func_refer returned error: No such file or directory ALSA lib conf.c:5036:(snd_config_expand) Args evaluate error: No such file or directory ALSA lib pcm.c:2565:(snd_pcm_open_noupdate) Unknown PCM bluealsa ALSA lib confmisc.c:1281:(snd_func_refer) Unable to find definition 'defaults.bluealsa.device' ALSA lib conf.c:4568:(_snd_config_evaluate) function snd_func_refer returned error: No such file or directory ALSA lib conf.c:5036:(snd_config_expand) Args evaluate error: No such file or directory ALSA lib pcm.c:2565:(snd_pcm_open_noupdate) Unknown PCM bluealsa Expression 'alsa_snd_pcm_hw_params_set_period_size_near( pcm, hwParams, &alsaPeriodFrames, &dir )' failed in 'src/hostapi/alsa/pa_linux_alsa.c', line: 924 ALSA lib pcm.c:2565:(snd_pcm_open_noupdate) Unknown PCM input Expression 'alsa_snd_pcm_hw_params_set_period_size_near( pcm, hwParams, &alsaPeriodFrames, &dir )' failed in 'src/hostapi/alsa/pa_linux_alsa.c', line: 924 Cannot connect to server socket err = No such file or directory Cannot connect to server request channel jack server is not running or cannot be started JackShmReadWritePtr::~JackShmReadWritePtr - Init not done for -1, skipping unlock JackShmReadWritePtr::~JackShmReadWritePtr - Init not done for -1, skipping unlock Expression 'paInvalidSampleRate' failed in 'src/hostapi/alsa/pa_linux_alsa.c', line: 2048 Expression 'PaAlsaStreamComponent_InitialConfigure( &self->playback, outParams, self->primeBuffers, hwParamsPlayback, &realSr )' failed in 'src/hostapi/alsa/pa_linux_alsa.c', line: 2722 Expression 'PaAlsaStream_Configure( stream, inputParameters, outputParameters, sampleRate, framesPerBuffer, &inputLatency, &outputLatency, &hostBufferSizeMode )' failed in 'src/hostapi/alsa/pa_linux_alsa.c', line: 2843 Traceback (most recent call last): File "detect-speaker.py", line 213, in <module> main() File "detect-speaker.py", line 128, in main output = True) # input��output�𓯎���True�ɂ��� File "/home/pi/.pyenv/versions/3.7.3/lib/python3.7/site-packages/pyaudio.py", line 750, in open stream = Stream(self, *args, **kwargs) File "/home/pi/.pyenv/versions/3.7.3/lib/python3.7/site-packages/pyaudio.py", line 441, in __init__ self._stream = pa.open(**arguments) OSError: [Errno -9997] Invalid sample rate

どうも、PyAudio にバグあるみたいで、Sample Rate=48000 でないと NG のようです。

... CHUNK=2**11 #RATE=44100 RATE=48000 p=pyaudio.PyAudio() stream=p.open(format = pyaudio.paInt16, channels = 1, rate = RATE, frames_per_buffer = CHUNK, input = False, #output_device_index = 0, output = True)

Raspi での実行の為に、別途、48K の WAV ファイルを用意するのが面倒なので、
simpleaudio を使う事にしました。
https://simpleaudio.readthedocs.io/en/latest/tutorial.html

detect-speaker2.py
# -*- coding:utf-8 -*- # Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """A demo that runs object detection on camera frames using OpenCV. TEST_DATA=../all_models Run coco model: python3 detect-speaker2.py \ --model ${TEST_DATA}/mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite \ --labels ${TEST_DATA}/coco_labels.txt """ import argparse import collections import common import cv2 import numpy as np import os from PIL import Image import re import tflite_runtime.interpreter as tflite # add by nishi from timeit import default_timer as timer import simpleaudio as sa import wave Object = collections.namedtuple('Object', ['id', 'score', 'bbox']) def load_labels(path): p = re.compile(r'\s*(\d+)(.+)') with open(path, 'r', encoding='utf-8') as f: lines = (p.match(line).groups() for line in f.readlines()) return {int(num): text.strip() for num, text in lines} class BBox(collections.namedtuple('BBox', ['xmin', 'ymin', 'xmax', 'ymax'])): """Bounding box. Represents a rectangle which sides are either vertical or horizontal, parallel to the x or y axis. """ __slots__ = () def get_output(interpreter, score_threshold, top_k, image_scale=1.0): """Returns list of detected objects.""" boxes = common.output_tensor(interpreter, 0) class_ids = common.output_tensor(interpreter, 1) scores = common.output_tensor(interpreter, 2) count = int(common.output_tensor(interpreter, 3)) def make(i): ymin, xmin, ymax, xmax = boxes[i] return Object( id=int(class_ids[i]), score=scores[i], bbox=BBox(xmin=np.maximum(0.0, xmin), ymin=np.maximum(0.0, ymin), xmax=np.minimum(1.0, xmax), ymax=np.minimum(1.0, ymax))) return [make(i) for i in range(top_k) if scores[i] >= score_threshold] def load_wave_tb(wav_dt): wav_tb={} for (id,path) in wav_dt.items(): wave_read = wave.open(path, 'rb') audio_data = wave_read.readframes(wave_read.getnframes()) num_channels = wave_read.getnchannels() bytes_per_sample = wave_read.getsampwidth() sample_rate = wave_read.getframerate() fs={'dt':audio_data,'num_channels':num_channels,'bytes_per_sample':bytes_per_sample,'sample_rate':sample_rate} wav_tb[id]=fs # cat=16 , dog=17 , cup=46 return wav_tb def main(): default_model_dir = '../all_models' default_model = 'mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite' default_labels = 'coco_labels.txt' parser = argparse.ArgumentParser() parser.add_argument('--model', help='.tflite model path', default=os.path.join(default_model_dir,default_model)) parser.add_argument('--labels', help='label file path', default=os.path.join(default_model_dir, default_labels)) parser.add_argument('--top_k', type=int, default=3, help='number of categories with highest score to display') parser.add_argument('--camera_idx', type=int, help='Index of which video source to use. ', default = 0) parser.add_argument('--threshold', type=float, default=0.5, help='classifier score threshold') args = parser.parse_args() # cat=16, dog=17, cup=46 wav_dt={16:'neko.wav',17:'inu.wav',46:'kop.wav'} wav_tb=load_wave_tb(wav_dt) if False: fs=wav_tb[46] play_obj = sa.play_buffer(fs['dt'], fs['num_channels'], fs['bytes_per_sample'], fs['sample_rate']) play_obj.wait_done() exit() print('Loading {} with {} labels.'.format(args.model, args.labels)) interpreter = common.make_interpreter(args.model) interpreter.allocate_tensors() if False: input_fs = interpreter.get_input_details() output_fs = interpreter.get_output_details() print('>input_fs=',input_fs) print('>output_fs=',output_fs) labels = load_labels(args.labels) cap = cv2.VideoCapture(args.camera_idx) # test by nishi for fps accum_time = 0 curr_fps = 0 fps = "FPS: ??" prev_time = timer() while cap.isOpened(): ret, frame = cap.read() if not ret: break cv2_im = frame cv2_im_rgb = cv2.cvtColor(cv2_im, cv2.COLOR_BGR2RGB) pil_im = Image.fromarray(cv2_im_rgb) common.set_input(interpreter, pil_im) interpreter.invoke() objs = get_output(interpreter, score_threshold=args.threshold, top_k=args.top_k) cv2_im = append_objs_to_img(cv2_im, objs, labels) # add by nishi for fps start curr_time = timer() exec_time = curr_time - prev_time prev_time = curr_time accum_time = accum_time + exec_time curr_fps = curr_fps + 1 if accum_time > 1: accum_time = accum_time - 1 fps = "FPS: " + str(curr_fps) curr_fps = 0 cv2.putText(cv2_im, text=fps, org=(3, 15), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=0.50, color=(255, 0, 0), thickness=2) #print('>type(cv2_im)=',type(cv2_im)) #print('>cv2_im.shape=',cv2_im.shape) #print('>cv2_im.dtype=',cv2_im.dtype) cv2.imshow('frame', cv2_im) if cv2.waitKey(1) & 0xFF == ord('q'): break obj_speaker(objs,wav_tb) cap.release() cv2.destroyAllWindows() def append_objs_to_img(cv2_im, objs, labels): height, width, channels = cv2_im.shape for obj in objs: x0, y0, x1, y1 = list(obj.bbox) x0, y0, x1, y1 = int(x0*width), int(y0*height), int(x1*width), int(y1*height) percent = int(100 * obj.score) label = '{}% {}'.format(percent, labels.get(obj.id, obj.id)) cv2_im = cv2.rectangle(cv2_im, (x0, y0), (x1, y1), (0, 255, 0), 2) cv2_im = cv2.putText(cv2_im, label, (x0, y0+30), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (255, 0, 0), 2) return cv2_im def obj_speaker(objs,wav_tb): for obj in objs: if obj.id in wav_tb: #print('>obj.id=',obj.id) fs=wav_tb[obj.id] play_obj = sa.play_buffer(fs['dt'], fs['num_channels'], fs['bytes_per_sample'], fs['sample_rate']) play_obj.wait_done() if __name__ == '__main__': main()

何とか、Raspi でも WAVがでるようになりましたが、出だしの音が聞こえないみたいです。
Windows10 だと問題無いのですが、...

後、Raspi  では、picamera を使うようにしました。
raspi-config で設定すれば、 picamera でも OpenCV でそのままアクセスできました。

但し、picamera だとバッファリングされて、先読みされている様で、
スピーカーで発声している時も、キャプチャーされた映像が溜まっているようで、
どうも感覚と一致しません。

スピーカーで発声し終わった時は、実際のカメラの前の映像は先に進んでいるのに、
その後も、バッファされた古い映像が出てくるので、ある意味、具合が悪いです。

Windows10 だと、発声している間は、映像が止まっていて、発声が終了した時の映像が、
次に画面に表示されるので、こちらの方が、感覚と一致するのではと思います。