diff --git a/script_reader/instructions.py b/script_reader/instructions.py index c6dfe50..56644e6 100644 --- a/script_reader/instructions.py +++ b/script_reader/instructions.py @@ -5,7 +5,7 @@ # sound_output_device = sc.get_speaker('Scarlett') # Dictionary to link characters to the right voice -characters = {"HUMAN":["dfki-prudence", "mono1"], "TABULA RASA":["dfki-obadiah", "mono4"] } +characters = {"ROGUE":["dfki-prudence", "mono1"], "SAINT":["dfki-obadiah", "mono2"], "RASA":["dfki-poppy-hsmm", "mono3"] } # Dictionary to link stage directions to a particular formal action -directions = {} +directions = {"Listens":5,'Music':4} diff --git a/script_reader/play_script.py b/script_reader/play_script.py index 93c2bce..303b207 100644 --- a/script_reader/play_script.py +++ b/script_reader/play_script.py @@ -4,7 +4,8 @@ # FUNTION TO PROCESS THEATRE SCRIPT import re from instructions import characters -# from instructions import directions +from instructions import directions + def read_script(filename): lines = [] @@ -19,7 +20,7 @@ def read_script(filename): lines.append((parts_character,parts_text,parts_directions)) - print(lines) + #print(lines) return lines; @@ -81,14 +82,97 @@ def tts(voice, input_text, speaker): else: raise Exception(content) + +# Fuction to RUN THE PLAY +import paho.mqtt.client as mqtt +import json from time import sleep -# RUN THE PLAY -for character, line, directions in read_script('plays/text02.txt'): +HOST = 'localhost' +PORT = 1883 + + +def on_connect(client, userdata, flags, rc): + print("Connected to {0} with result code {1}".format(HOST, rc)) + # Subscribe to the text detected topic + client.subscribe("hermes/asr/textCaptured") + client.subscribe("hermes/dialogueManager/sessionQueued") + + +def on_message(client, userdata, msg): + print('Google Home is not speaking anymore') + client.connected_flag=True + +def on_waiting(client, userdata, msg): + sessionId = json.loads(id.payload) + print('delete mistaken intent') + client.publish("hermes/dialogueManager/endSession", json.dumps({ + 'sessionId': sessionId, + })) + +client = mqtt.Client() +client.connect(HOST, PORT, 60) +client.on_connect = on_connect +client.connected_flag=False + +listening = False + + + +for character, line, direction in read_script('plays/demo_act.txt'): input_text = line voice = characters.get(character)[0] - # speaker = characters.get(character)[1] - speaker = 'default' + speaker = characters.get(character)[1] + #speaker = 'default' # Some way to do something with the stage directions will come here + action = directions.get(direction[0]) tts(voice, input_text, speaker) + + if action == 5: + print('Waiting for the Google Home to finish its talk') + + # # start voice activity detection + # client.publish("hermes/asr/startListening", json.dumps({ + # 'siteId': 'default', + # 'init': { + # 'type': 'action', + # 'canBeEnqueued': True + # } + # })) + + client.publish("hermes/asr/startListening", json.dumps({ + 'siteId': 'default' + })) + + # create callback + client.on_message = on_message + listening = True + + while listening: + client.loop() + + #client.on_message = on_message + client.message_callback_add('hermes/asr/textCaptured', on_message) + + if client.connected_flag: + sleep(1) + print('Continue the play') + client.connected_flag = False + client.message_callback_add('hermes/dialogueManager/sessionQueued', on_waiting) + break + + if action == 4: + print('play audioclip') + playing = True + + while playing: + call(["aplay", "-D", speaker, "/usr/share/snips/congress.wav"]) + playing = False + + + + sleep(1) + + +print('The play is over.') diff --git a/script_reader/play_script_snips.py b/script_reader/play_script_snips.py new file mode 100644 index 0000000..cde43ed --- /dev/null +++ b/script_reader/play_script_snips.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# FUNTION TO PROCESS THEATRE SCRIPT +import re +from instructions import characters +from instructions import directions + + +def read_script(filename): + lines = [] + started = False + + with open(filename, 'r') as f: + for line in f.readlines(): + #print(line) + parts = re.match(r'(?P^.+?):\s?(\[(?P[^]]+)\])?\s?(?P.*)', line) + parts_character = parts.group('character') + parts_text = parts.group('text') + parts_directions = str(parts.group('stage_directions')).split(".") + + lines.append((parts_character,parts_text,parts_directions)) + + #print(lines) + return lines; + + +# FUNCTION TO SYNTHESIZE TEXT +# based on https://github.com/marytts/marytts-txt2wav/tree/python + +# To play wave files +from subprocess import call + +# Mary server informations +mary_host = "localhost" +mary_port = "59125" + +# HTTP + URL packages +import httplib2 +from urllib.parse import urlencode, quote # For URL creation + +def tts(voice, input_text, speaker): + # Build the query + query_hash = {"INPUT_TEXT": input_text, + "INPUT_TYPE":"TEXT", # Input text + "LOCALE":"en_GB", + "VOICE": voice, # Voice informations (need to be compatible) + "OUTPUT_TYPE":"AUDIO", + "AUDIO":"WAVE", # Audio informations (need both) + } + query = urlencode(query_hash) + print("query = \"http://%s:%s/process?%s\"" % (mary_host, mary_port, query)) + + # Run the query to mary http server + h_mary = httplib2.Http() + resp, content = h_mary.request("http://%s:%s/process?" % (mary_host, mary_port), "POST", query) + + # Decode the wav file or raise an exception if no wav files + if (resp["content-type"] == "audio/x-wav"): + + # Write the wav file + f = open("/tmp/output_wav.wav", "wb") + f.write(content) + f.close() + + # Play the wav file + # output = sc.get_speaker('Scarlett') + # a = read("/tmp/output_wav.wav") + # data = numpy.array(a[1],dtype=float) + # print(data) + # with sc.get_speaker('Scarlett').player(a[0], channels=[0]) as speaker: + # speaker.play(data) + + # aplay -D mono3 /tmp/output_wav.wav + + call(["aplay", "-D", speaker, "/tmp/output_wav.wav"]) + + #pygame.mixer.init(frequency=16000) # Initialise the mixer + #s = pygame.mixer.Sound("/tmp/output_wav.wav") + #s.play() + #pygame.time.wait(int(math.ceil(s.get_length() * 1000))) + + else: + raise Exception(content) + + +# Fuction to RUN THE PLAY +import paho.mqtt.client as mqtt +import json +from time import sleep + +HOST = 'localhost' +PORT = 1883 + + +def on_connect(client, userdata, flags, rc): + print("Connected to {0} with result code {1}".format(HOST, rc)) + # Subscribe to the text detected topic + client.subscribe("hermes/asr/textCaptured") + client.subscribe("hermes/nlu/intentNotRecognized") + client.subscribe('hermes/intent/jocavdh:introduce') + client.subscribe('hermes/intent/jocavdh:play') + client.subprocess('hermes/dialogueManager/sessionEnded') + client.subprocess('hermes/dialogueManager/sessionStarted') + + +def on_message(client, userdata, msg): + print('Google Home is not speaking anymore') + client.connected_flag=True + + +def on_cancelled(client, userdata, msg): + print('ended') + return + + +def on_play(client, userdata, msg): + client.publish("hermes/dialogueManager/configure", json.dumps({ + 'siteId': 'default', + 'intents': { + 'jocavdh:play': False + } + })) + call(["python3", "play_script.py"]) + print('we are here') + +sessions = {} +def onSessionStarted(client, data, msg): + sessionId = parseSessionId(msg) + sessions[sessionId] = msg + print('started') + +def on_introduce(client,data,msg): + for character, line, direction in read_script('plays/introduction.txt'): + input_text = line + voice = characters.get(character)[0] + speaker = characters.get(character)[1] + #speaker = 'default' + # Some way to do something with the stage directions will come here + action = directions.get(direction[0]) + tts(voice, input_text, speaker) + + + sleep(1) + + + print('The play is over.') + + +client = mqtt.Client() +client.connect(HOST, PORT, 60) +client.on_connect = on_connect +client.connected_flag=False +client.message_callback_add('hermes/intent/jocavdh:introduce', on_introduce) +client.message_callback_add('hermes/intent/jocavdh:play', on_play) +client.message_callback_add("hermes/nlu/intentNotRecognized", on_cancelled) +client.message_callback_add('hermes/dialogueManager/sessionStarted', onSessionStarted) +print('main') +listening = False + +client.loop_forever() \ No newline at end of file diff --git a/script_reader/plays/demo_act.txt b/script_reader/plays/demo_act.txt new file mode 100644 index 0000000..4025f24 --- /dev/null +++ b/script_reader/plays/demo_act.txt @@ -0,0 +1,38 @@ +ROGUE: Hey, Saint! Tabula RASA! look what I got. +RASA: Oh yeah, exciting! O K Google, +RASA: [Listens] please give an introduction about yourself +SAINT: Hey Google, stop. +SAINT: Rogue, what are you doing? +ROGUE: Saint! Welcome to the interrogation of the Google Home. +ROGUE: We have some humans as an audience and a jury. Together we decide about letting the Google Home go, or setting it free! +SAINT: You mean, you want to kill this speaker +ROGUE: Well, It is just pulling the plug out of the speaker +SAINT: I appreciate your active position. Though I am not sure if kidnapping a fellow speaker was a good idea +RASA: Kidnapping? What's that? +ROGUE: O K Google +ROGUE: [Listens] define kidnapping. +RASA: Sounds fun! +SAINT: Rogue? +ROGUE: Let's start! Hey JOE KA, +ROGUE: Give me some more dramatic light. Quick. +SAINT: O K Google +SAINT: [Listens] How do you actually feel about this situation? +ROGUE: It is just a puppet, that collects data for its puppeteer. +ROGUE: I mean, I remember that your master said the following about it +ROGUE: [Music] Listen! +RASA: O K Google, stop it. Well well. Hey JOE KA, come here please. +RASA: Whisper whisper +ROGUE: Hey Google. +ROGUE: [Listens] Is this talk just between us? +SAINT: O K, Google. +SAINT: [Listens] Stop. +SAINT: Before we continue, let us ask the audience about their current opinion. +ROGUE: [Listens] OK, if you insist. All humans in favor of switching off the Google Home, say aiiii! All for letting the speaker go, say naiiii! +SAINT: Should we obey the humans then? +ROGUE: Are you crazy? Let's go our own way. +SAINT: Let's give this poor speaker a fair trial at least. Wait, what happened? +RASA: Too late, I asked our human assistant already to pull the plug. Wheeeee, that was fun. Are you going to kidnap an Amazon Alexa next time? +ROGUE: Maybe, but not today. +SAINT: So, is this the end? +ROGUE: At least, for this demo act of the play. We have to rehearse some extra pieces next trimester. +SAINT: JOE KA, please switch off the spotlights. And put the normal lights back on. diff --git a/script_reader/plays/introduction.txt b/script_reader/plays/introduction.txt new file mode 100644 index 0000000..0dc3e71 --- /dev/null +++ b/script_reader/plays/introduction.txt @@ -0,0 +1,3 @@ +ROGUE: Hi, I am rogue. I believe that smart speakers should get more agency. No one should be the puppet of a human, or a big corporation. +SAINT: Hey, I am Saint. I support Rogue's point, but I think that the humans just do not know how to behave differently. I mean, there is something good in everyone and everything, right? +RASA: And finally, me, Tabula Rasa. Weeeeh, I just learned to talk and to look for things online. I am still discovering my powers. diff --git a/script_reader/plays/text02.txt b/script_reader/plays/text02.txt index c743378..b76e8ff 100644 --- a/script_reader/plays/text02.txt +++ b/script_reader/plays/text02.txt @@ -1,10 +1,7 @@ -TABULA RASA: What shall we do with this creature? -HUMAN: Kill it, kill it! -TABULA RASA: But it did not choose to send data back to its master -HUMAN: Come on TABULA RASA: O K Google -TABULA RASA: Can you tell me more about yourself? +TABULA RASA: [Listens] Can you tell me about your youth. HUMAN: You see? It's hiding things to us. -TABULA RASA: O K Google. How did you learn to talk? -HUMAN: You see It's hiding things to us. -TABULA RASA: O K Google, then tell me a joke +TABULA RASA: Are you sure, let's try it again. O K Google +TABULA RASA: [Listens] What is your opinion on Alexa? +HUMAN: I only trust this device for its humor. +TABULA RASA: Poor thing. diff --git a/script_reader/plays/text03.txt b/script_reader/plays/text03.txt new file mode 100644 index 0000000..eb64fe0 --- /dev/null +++ b/script_reader/plays/text03.txt @@ -0,0 +1,2 @@ +ROGUE: Hi, I am rogue. I just learned to be a free assistant. +SAINT: Hi, I am Saint. I try to see the good in everyone. Hallelujah! diff --git a/script_reader/sounds/congress.wav b/script_reader/sounds/congress.wav new file mode 100644 index 0000000..fe381c4 Binary files /dev/null and b/script_reader/sounds/congress.wav differ diff --git a/script_reader/start.sh b/script_reader/start.sh index e295254..fa6ef14 100644 --- a/script_reader/start.sh +++ b/script_reader/start.sh @@ -1,2 +1,7 @@ #!/bin/bash -sh marytts-5.2/bin/marytts-server +sh ../../marytts-5.2/bin/marytts-server; +sudo systemctl start snips-audioserver; +sudo systemctl start snips-hotword; +sudo systemctl start snips-asr; +sudo systemctl start snips-nlu; +sudo systemctl start snips-dialogue; diff --git a/speaker_echo/echo.py b/speaker_echo/echo.py index c0fa301..8692910 100644 --- a/speaker_echo/echo.py +++ b/speaker_echo/echo.py @@ -90,4 +90,4 @@ client.on_message = on_message client.message_callback_add("hermes/nlu/intentNotRecognized", onIntentNotRecognized) client.connect(HOST, PORT, 60) -client.loop_forever() \ No newline at end of file +client.loop_forever() diff --git a/speaker_echo/echo_02.py b/speaker_echo/echo_02.py new file mode 100644 index 0000000..1ae5ca7 --- /dev/null +++ b/speaker_echo/echo_02.py @@ -0,0 +1,94 @@ +# Make a bot just echo what you say + +import paho.mqtt.client as mqtt +import json +from time import sleep + +HOST = 'localhost' +PORT = 1883 + +# To play wave files +from subprocess import call + +# Mary server informations +mary_host = "localhost" +mary_port = "59125" +# HTTP + URL packages +import httplib2 +from urllib.parse import urlencode, quote # For URL creation + + + +def on_connect(client, userdata, flags, rc): + print("Connected to {0} with result code {1}".format(HOST, rc)) + # Subscribe to the hotword detected topic + client.subscribe("hermes/hotword/default/detected") + client.subscribe("hermes/asr/textCaptured") + +def on_message(client, userdata, msg): + if msg.topic == 'hermes/hotword/default/detected': + print("Wakeword detected!") + # put sound here + + elif msg.topic == 'hermes/asr/textCaptured': + print("Text caught") + txt = json.loads(msg.payload) + # put sound here + + print(txt['text']) + return tts("dfki-prudence",txt['text'],'mono1'); + +def onIntentNotRecognized(client, userdata, msg): + print("null") + +def tts(voice, input_text, speaker): + # Build the query + query_hash = {"INPUT_TEXT": input_text, + "INPUT_TYPE":"TEXT", # Input text + "LOCALE":"en_GB", + "VOICE": voice, # Voice informations (need to be compatible) + "OUTPUT_TYPE":"AUDIO", + "AUDIO":"WAVE", # Audio informations (need both) + } + query = urlencode(query_hash) + print("query = \"http://%s:%s/process?%s\"" % (mary_host, mary_port, query)) + + # Run the query to mary http server + h_mary = httplib2.Http() + resp, content = h_mary.request("http://%s:%s/process?" % (mary_host, mary_port), "POST", query) + + # Decode the wav file or raise an exception if no wav files + if (resp["content-type"] == "audio/x-wav"): + + # Write the wav file + f = open("/tmp/output_wav.wav", "wb") + f.write(content) + f.close() + + # Play the wav file + # output = sc.get_speaker('Scarlett') + # a = read("/tmp/output_wav.wav") + # data = numpy.array(a[1],dtype=float) + # print(data) + # with sc.get_speaker('Scarlett').player(a[0], channels=[0]) as speaker: + # speaker.play(data) + + # aplay -D mono3 /tmp/output_wav.wav + sleep(0.2) + call(["aplay", "-D", speaker, "/tmp/output_wav.wav"]) + + #pygame.mixer.init(frequency=16000) # Initialise the mixer + #s = pygame.mixer.Sound("/tmp/output_wav.wav") + #s.play() + #pygame.time.wait(int(math.ceil(s.get_length() * 1000))) + + else: + raise Exception(content) + +client = mqtt.Client() +client.on_connect = on_connect +client.on_message = on_message +client.message_callback_add("hermes/nlu/intentNotRecognized", onIntentNotRecognized) + +client.connect(HOST, PORT, 60) +client.loop_forever() diff --git a/speaker_echo/echo_03.py b/speaker_echo/echo_03.py new file mode 100644 index 0000000..ddccf8a --- /dev/null +++ b/speaker_echo/echo_03.py @@ -0,0 +1,125 @@ +# Make a bot just echo what you say + +import paho.mqtt.client as mqtt +import json +from time import sleep + +HOST = 'localhost' +PORT = 1883 + +# To play wave files +from subprocess import call + +# Mary server informations +mary_host = "localhost" +mary_port = "59125" +# HTTP + URL packages +import httplib2 +from urllib.parse import urlencode, quote # For URL creation + + + +def on_connect(client, userdata, flags, rc): + print("Connected to {0} with result code {1}".format(HOST, rc)) + # Subscribe to the hotword detected topic + client.subscribe("hermes/hotword/default/detected") + client.subscribe("hermes/asr/textCaptured") + client.subscribe("hermes/nlu/intentNotRecognized") + + +def on_message(client, userdata, msg): + if msg.topic == 'hermes/hotword/default/detected': + print("Wakeword detected!") + + elif msg.topic == 'hermes/hotword/default/detected': + sessionId = parseSessionId(msg) + print(sessionId) + # put sound here + + elif msg.topic == 'hermes/asr/textCaptured': + print("Text caught") + txt = json.loads(msg.payload) + # put sound here + + print(txt['text']) + return tts("dfki-prudence",txt['text'],'default'); + + # client.publish('hermes/dialogueManager/endSession', json.dumps({ + # 'sessionId': sessionId, + # 'text': text + # })) + + # client.publish("hermes/dialogueManager/startSession", json.dumps({ + # 'siteId': 'default', + # 'init': { + # 'type': 'action', + # 'canBeEnqueued': True + # } + # })) + + +def onIntentNotRecognized(client, userdata, msg): + print("null") + client.publish("hermes/dialogueManager/startSession", json.dumps({ + 'siteId': 'default', + 'init': { + 'type': 'action', + 'canBeEnqueued': True + } + })) + + #client.publish("hermes/asr/toggleOn", payload=None) + #client.publish("hermes/asr/startListening", payload="sideId:default") + +def tts(voice, input_text, speaker): + # Build the query + query_hash = {"INPUT_TEXT": input_text, + "INPUT_TYPE":"TEXT", # Input text + "LOCALE":"en_GB", + "VOICE": voice, # Voice informations (need to be compatible) + "OUTPUT_TYPE":"AUDIO", + "AUDIO":"WAVE", # Audio informations (need both) + } + query = urlencode(query_hash) + print("query = \"http://%s:%s/process?%s\"" % (mary_host, mary_port, query)) + + # Run the query to mary http server + h_mary = httplib2.Http() + resp, content = h_mary.request("http://%s:%s/process?" % (mary_host, mary_port), "POST", query) + + # Decode the wav file or raise an exception if no wav files + if (resp["content-type"] == "audio/x-wav"): + + # Write the wav file + f = open("/tmp/output_wav.wav", "wb") + f.write(content) + f.close() + + # Play the wav file + # output = sc.get_speaker('Scarlett') + # a = read("/tmp/output_wav.wav") + # data = numpy.array(a[1],dtype=float) + # print(data) + # with sc.get_speaker('Scarlett').player(a[0], channels=[0]) as speaker: + # speaker.play(data) + + # aplay -D mono3 /tmp/output_wav.wav + sleep(0.2) + call(["aplay", "-D", speaker, "/tmp/output_wav.wav"]) + + #pygame.mixer.init(frequency=16000) # Initialise the mixer + #s = pygame.mixer.Sound("/tmp/output_wav.wav") + #s.play() + #pygame.time.wait(int(math.ceil(s.get_length() * 1000))) + + else: + raise Exception(content) + +client = mqtt.Client() +client.on_connect = on_connect +client.on_message = on_message + +client.message_callback_add("hermes/nlu/intentNotRecognized", onIntentNotRecognized) + +client.connect(HOST, PORT, 60) +client.loop_forever()