diff --git a/.gitignore b/.gitignore index b36ff80..78c8e31 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ __pycache__/ # Distribution / packaging .Python build/ +marytts develop-eggs/ dist/ downloads/ diff --git a/01_start_tts b/01_start_tts new file mode 100755 index 0000000..141588c --- /dev/null +++ b/01_start_tts @@ -0,0 +1,3 @@ +#!/bin/bash + +marytts/bin/marytts-server \ No newline at end of file diff --git a/02_start_script b/02_start_script new file mode 100755 index 0000000..76992f1 --- /dev/null +++ b/02_start_script @@ -0,0 +1,3 @@ +#!/bin/bash + +sudo ./smart_speaker_theatre.py \ No newline at end of file diff --git a/03_debug b/03_debug new file mode 100755 index 0000000..ce934b4 --- /dev/null +++ b/03_debug @@ -0,0 +1,3 @@ +#!/bin/bash + +snips-watch -v \ No newline at end of file diff --git a/README.md b/README.md old mode 100755 new mode 100644 index 20abfd5..e211a44 --- a/README.md +++ b/README.md @@ -1,5 +1,20 @@ -# Smart speaker theatre backend +# Script reader -Scripts which turn detected user intents in actions done by the speakers. +This programme reads out a theatre script, using the speech synthesizer Mary TTS. -Work in progress. +Install mary-tts following the instructions in the pdf and start it +then run the play_script.py + + +---- +Writing new plays + +As an input, write a play in the following format: + +CHARACTERNAME: [ stage directions ] text to say + +Put the script in the plays directory, and put the filename in play_script.py. + +Use instructions.py to set-up the characters and the voices. + +Stage directions (for example lights, silences etc.) are still in development diff --git a/play_interrogation_act.py b/act.py similarity index 59% rename from play_interrogation_act.py rename to act.py index c467b28..19dacb3 100644 --- a/play_interrogation_act.py +++ b/act.py @@ -1,12 +1,24 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +# PLAY_ACT.py +# This script runs the play +# It is in a seperate file to enable the mechanism to detect the Google Home speaking, before continuing to the next line + # Libraries +from config import characters, directions +from logic import tts, read_script +from pixel_ring import pixel_ring +from subprocess import call import paho.mqtt.client as mqtt import json +import sys from time import sleep -from logic import tts, read_script -from config import characters, directions + +# Switch of LED's of speakers at the start of the play +pixel_ring.off() + + # === SETUP OF MQTT PART 1 === @@ -14,9 +26,6 @@ from config import characters, directions HOST = 'localhost' PORT = 1883 - -# === FUNCTIONS THAT ARE TRIGGERED WHEN AN INTENT IS DETECTED === - # Subscribe to relevant MQTT topics def on_connect(client, userdata, flags, rc): print("Connected to {0} with result code {1}".format(HOST, rc)) @@ -24,35 +33,55 @@ def on_connect(client, userdata, flags, rc): client.subscribe("hermes/asr/textCaptured") client.subscribe("hermes/dialogueManager/sessionQueued") -# Set Flag when Google Home is done speaking -def on_message(client, userdata, msg): +# Function which sets a flag when the Google Home is not speaking +# Callback of MQTT message that says that the text is captured by the speech recognition (ASR) +def done_speaking(client, userdata, msg): print('Google Home is not speaking anymore') client.connected_flag=True -# Remove any detected intents that are activated by the speaking Google Home -def on_waiting(client, userdata, msg): +# Function which removes intents that are by accident activated by the Google Home +# e.g. The google home says introduce yourself, which could trigger the other speakers to introduce themselves +# Snips works with queing of sessions, so this situation would only happen after this play is finished +def remove_sessions(client, userdata, msg): sessionId = json.loads(id.payload) print('delete mistaken intent') client.publish("hermes/dialogueManager/endSession", json.dumps({ 'sessionId': sessionId, })) + + + +# === SETUP OF MQTT PART 2 === + +# Initialise MQTT client client = mqtt.Client() client.connect(HOST, PORT, 60) client.on_connect = on_connect -client.connected_flag=False + + + +# === Read script and run the play === + +# Flags to check if the system is listening, or not +client.connected_flag=False listening = False +# Read the script and run the play + -for character, line, direction in read_script('play_scripts/interruption_02.txt'): +file = sys.argv[1] # get the chosen act passed by smart_speaker_theatre.py + +for character, line, direction in read_script(file): input_text = line voice = characters.get(character)[0] speaker = characters.get(character)[1] #speaker = 'default' # Some way to do something with the stage directions will come here action = directions.get(direction[0]) + pixel_ring.speak() tts(voice, input_text, speaker) if action == 'listen_google_home': @@ -67,28 +96,32 @@ for character, line, direction in read_script('play_scripts/interruption_02.txt' # } # })) + # Activate the microphone and speech recognition client.publish("hermes/asr/startListening", json.dumps({ 'siteId': 'default' })) + # LED to listening mode + pixel_ring.listen() + # create callback - client.on_message = on_message + client.on_message = done_speaking listening = True while listening: client.loop() #client.on_message = on_message - client.message_callback_add('hermes/asr/textCaptured', on_message) + client.message_callback_add('hermes/asr/textCaptured', done_speaking) if client.connected_flag: sleep(1) print('Continue the play') client.connected_flag = False - client.message_callback_add('hermes/dialogueManager/sessionQueued', on_waiting) + client.message_callback_add('hermes/dialogueManager/sessionQueued', remove_sessions) break - if action == 4: + if action == 'music': print('play audioclip') playing = True @@ -98,8 +131,8 @@ for character, line, direction in read_script('play_scripts/interruption_02.txt' - - sleep(1) + pixel_ring.off() # Switch of the lights when done speaking + sleep(1) # Add a short pause between the lines -print('The act is over.') \ No newline at end of file +print('The act is done.') \ No newline at end of file diff --git a/action-play.py b/action-play.py deleted file mode 100755 index d28ef5a..0000000 --- a/action-play.py +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -# Libraries -import paho.mqtt.client as mqtt -import json -from time import sleep -from logic import tts, read_script -from config import characters, directions - - -# === SETUP OF MQTT PART 1 === - -# Location of the MQTT server -HOST = 'localhost' -PORT = 1883 - -# Subscribe to relevant MQTT topics -def on_connect(client, userdata, flags, rc): - print("Connected to {0} with result code {1}".format(HOST, rc)) - client.subscribe('hermes/intent/jocavdh:play_intro_act') # to check for intent to play the act - client.subscribe('hermes/intent/jocavdh:question_continue_act') # to check for the intent to continue to the next act - client.subscribe("hermes/nlu/intentNotRecognized") # to check if the speaker didn't understand the user, trigger for fallback function - - -# === FUNCTIONS THAT ARE TRIGGERED WHEN AN INTENT IS DETECTED === - -# Function which is triggered when the intent play_intro_act is activated -def on_play_intro_act(client,data,msg): - data = json.loads(msg.payload) - sessionId = data['sessionId'] - - script_lines = read_script('play_scripts/demo.txt') # pick a random introduction script - - for character, line, direction in script_lines: - input_text = line - voice = characters.get(character)[0] - speaker = characters.get(character)[1] - #speaker = 'default' - # Some way to do something with the stage directions will come here - action = directions.get(direction[0]) - tts(voice, input_text, speaker) - print('say this sentence') - - # if action == 'start_interrogation': - # call(["python3", "play_interrogation_act.py"]) - - - if action == 'listen_audience': - print('listen to the audience') - - client.publish('hermes/dialogueManager/endSession', json.dumps({ - 'sessionId': sessionId - })) - - client.publish('hermes/dialogueManager/startSession', json.dumps({ - 'siteId': 'default', - 'init': {'type': 'action', 'canBeEnqueued': True, 'intentFilter':['jocavdh:question_continue_act']} - })) - - break - -# Function which is triggered when the intent question_continue_act is activated -def on_question_continue_act(client,data,msg): - - data = json.loads(msg.payload) - answer_value = data['slots'][0]['value']['value'] - print(answer_value) - - voice = "dfki-obadiah" - speaker = 'default' - - if answer_value == 'no': - print('no') - - if answer_value == 'yes': - call(["python3", "play_interrogation_act.py"]) - - print('The play is over.') - -# Function which is triggered when no intent is recognized -def onIntentNotRecognized(client, data, msg): - data = json.loads(msg.payload) - line_number = 1 - print('We continue on line') - print(line_number) - - on_introduce(client,data,msg, line_number) - - -# === SETUP OF MQTT PART 2 === - -# Initialise MQTT client -client = mqtt.Client() -client.connect(HOST, PORT, 60) -client.on_connect = on_connect - -# Connect each MQTT topic to which you subscribed to a handler function -client.message_callback_add('hermes/intent/jocavdh:play_intro_act', on_play_intro_act) -client.message_callback_add('hermes/intent/jocavdh:question_continue_act', on_question_continue_act) -client.message_callback_add("hermes/nlu/intentNotRecognized", onIntentNotRecognized) - -# Keep checking for new MQTT messages -client.loop_forever() \ No newline at end of file diff --git a/config.py b/config.py old mode 100755 new mode 100644 index f1d9758..ba22a45 --- a/config.py +++ b/config.py @@ -1,22 +1,14 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# play_config.py +# config.py # This script contains the basic configuration of the play # A list of the characters and their voices # A list of stage directions which connect to formal instructions for the hardware # --- -# Define sound output device here -# sound_output_device = sc.get_speaker('Scarlett') - # Dictionary to link characters to the right voice -characters = {"ROGUE":["dfki-prudence", "mono1"], "SAINT":["dfki-obadiah", "mono2"], "RASA":["dfki-poppy-hsmm", "mono3"] } +characters = {"ROGUE":["cmu-slt-hsmm", "mono2"], "SAINT":["dfki-obadiah-hsmm", "mono3"], "RASA":["dfki-poppy-hsmm", "mono1"] } # Dictionary to link stage directions to a particular formal action -directions = { - 'Ask audience':'listen_audience', - 'Start interrogation':'start_interrogation', - 'Listens to Google Home':'listen_google_home', - 'Music':'audio' -} +directions = {"Listen to Google Home":'listen_google_home','Music':'music'} diff --git a/logic.py b/logic.py index e1df1b9..1b1cbd2 100644 --- a/logic.py +++ b/logic.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- -# play_logic.py +# logic.py # This script contains the logic to turn the play_script into instructions for the hardware # --- @@ -46,10 +46,19 @@ import os from urllib.parse import urlencode, quote # For URL creation def tts(voice, input_text, speaker): + + if speaker == "mono3": + volume_level = "amount:0.5" + + else: + volume_level = "amount:1.0" + # Build the query query_hash = {"INPUT_TEXT": input_text, "INPUT_TYPE":"TEXT", # Input text "LOCALE":"en_GB", + "effect_VOLUME_selected":"on", + "effect_VOLUME_parameters":volume_level, "VOICE": voice, # Voice informations (need to be compatible) "OUTPUT_TYPE":"AUDIO", "AUDIO":"WAVE", # Audio informations (need both) @@ -70,10 +79,8 @@ def tts(voice, input_text, speaker): f.write(content) f.close() - # aplay -D mono3 /tmp/output_wav.wav - #call(["aplay", "-D", "sysdefault:CARD=ArrayUAC10", fpath]) - call(["aplay", fpath]) + call(["aplay", "-D", speaker, "/tmp/output_wav.wav"]) else: diff --git a/play_scripts/demo.txt b/play_scripts/demo.txt index eabab40..4a080fa 100644 --- a/play_scripts/demo.txt +++ b/play_scripts/demo.txt @@ -1 +1,3 @@ -SAINT: [Ask audience] do you want to continue? \ No newline at end of file +ROGUE: Do you want to continue? +RASA: Well, I definitely want to +SAINT: So do I diff --git a/play_scripts/interruption.txt b/play_scripts/interruption.txt index d4d9b70..8c1cfbb 100755 --- a/play_scripts/interruption.txt +++ b/play_scripts/interruption.txt @@ -1,2 +1,2 @@ SAINT: Yes, I am ready to go. But first tell a bit more about that silly project of yours. -SAINT: [Wait for audience] Sorry, I gonna let you finish, but do you mind if I introduce myself first? +SAINT: Sorry, I gonna let you finish, but do you mind if I introduce myself first? diff --git a/requirements.txt b/requirements.txt deleted file mode 100755 index a7020c9..0000000 --- a/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -# Needed for Snips platform 1.1.0 (0.61.1) -hermes-python>=0.3.3 -toml diff --git a/restart b/restart deleted file mode 100644 index a69a009..0000000 --- a/restart +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -sudo systemctl restart snips-skill-server -v & -sudo systemctl restart snips-watch -v \ No newline at end of file diff --git a/setup.sh b/setup.sh deleted file mode 100755 index e7ea6c4..0000000 --- a/setup.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -set -e - -# Copy config.ini.default if it exists and config.ini doesn't exist. -if [ -e config.ini.default ] && [ ! -e config.ini ]; then - cp config.ini.default config.ini - chmod a+w config.ini -fi - -PYTHON=$(command -v python3) -VENV=venv - -if [ -f "$PYTHON" ]; then - - if [ ! -d $VENV ]; then - # Create a virtual environment if it doesn't exist. - $PYTHON -m venv $VENV - else - if [ -e $VENV/bin/python2 ]; then - # If a Python2 environment exists, delete it first - # before creating a new Python 3 virtual environment. - rm -r $VENV - $PYTHON -m venv $VENV - fi - fi - - # Activate the virtual environment and install requirements. - # shellcheck disable=SC1090 - . $VENV/bin/activate - pip3 install -r requirements.txt - -else - >&2 echo "Cannot find Python 3. Please install it." -fi diff --git a/smart_speaker_theatre.py b/smart_speaker_theatre.py new file mode 100755 index 0000000..56ea055 --- /dev/null +++ b/smart_speaker_theatre.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# SMART SPEAKER THEATRE +# This script reads the triggers for activated user intents sent by Snips over MQTT +# Using this triggers, it will start particular actions and scripts + + +# Libraries +import re +from config import characters, directions +from logic import tts, read_script +from subprocess import call +import paho.mqtt.client as mqtt +import json +from time import sleep +from pixel_ring import pixel_ring + + +# === SETUP OF MQTT PART 1 === + +# Location of the MQTT server +HOST = 'localhost' +PORT = 1883 + +# Subscribe to relevant MQTT topics +def on_connect(client, userdata, flags, rc): + print("Connected to {0} with result code {1}".format(HOST, rc)) + client.subscribe('hermes/intent/jocavdh:play_intro_act') # to check for intent to play the act + client.subscribe('hermes/intent/jocavdh:question_continue_act') # to check for the intent to continue to the next act + client.subscribe('hermes/hotword/default/detected') + + + + +# === FUNCTIONS THAT ARE TRIGGERED WHEN AN INTENT IS DETECTED === + +def on_wakeword(client, userdata, msg): + pixel_ring.think() + +# Function which is triggered when the intent play_intro_act is activated +def on_play_act(client, userdata, msg): + + + # # disable this intent to avoid playing another act triggered by the Google Home + # client.publish("hermes/dialogueManager/configure", json.dumps({ + # 'siteId': 'default', + # 'intents': { + # 'jocavdh:play': False + # } + # })) + + call(["python3", "act.py", "play_scripts/demo.txt"]) + +# Function which is triggered when the intent introduction is activated +def on_play_introduction(client,data,msg): + + for character, line, direction in read_script('plays/introduction.txt'): + input_text = line + voice = characters.get(character)[0] + speaker = characters.get(character)[1] + action = directions.get(direction[0]) + tts(voice, input_text, speaker) + sleep(1) # add a pause between each line + + + print('The act is over.') + + + +# === SETUP OF MQTT PART 2 === + +# Initialise MQTT client +client = mqtt.Client() +client.connect(HOST, PORT, 60) +client.on_connect = on_connect + +# Connect each MQTT topic to which you subscribed to a handler function +client.message_callback_add('hermes/intent/jocavdh:play_intro_act', on_play_act) +client.message_callback_add('hermes/hotword/default/detected', on_wakeword) + +# Keep checking for new MQTT messages +client.loop_forever() \ No newline at end of file diff --git a/sounds/congress.wav b/sounds/congress.wav new file mode 100644 index 0000000..fe381c4 Binary files /dev/null and b/sounds/congress.wav differ diff --git a/start b/start deleted file mode 100755 index 5c71c98..0000000 --- a/start +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -snips-skill-server -v & -snips-watch -v \ No newline at end of file diff --git a/start-tts b/start-tts deleted file mode 100644 index 39b83e6..0000000 --- a/start-tts +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -~/marytts/bin/marytts-server \ No newline at end of file diff --git a/tmp/.placeholder b/tmp/.placeholder deleted file mode 100755 index e69de29..0000000