simpler and cleaned up backend including handy debugging scripts

new_voices
jocavdh 5 years ago
parent eb666374d6
commit be4a79c3e9

1
.gitignore vendored

@ -29,6 +29,7 @@ __pycache__/
# Distribution / packaging
.Python
build/
marytts
develop-eggs/
dist/
downloads/

@ -0,0 +1,3 @@
#!/bin/bash
marytts/bin/marytts-server

@ -0,0 +1,3 @@
#!/bin/bash
sudo ./smart_speaker_theatre.py

@ -0,0 +1,3 @@
#!/bin/bash
snips-watch -v

@ -1,5 +1,20 @@
# Smart speaker theatre backend
# Script reader
Scripts which turn detected user intents in actions done by the speakers.
This programme reads out a theatre script, using the speech synthesizer Mary TTS.
Work in progress.
Install mary-tts following the instructions in the pdf and start it
then run the play_script.py
----
Writing new plays
As an input, write a play in the following format:
CHARACTERNAME: [ stage directions ] text to say
Put the script in the plays directory, and put the filename in play_script.py.
Use instructions.py to set-up the characters and the voices.
Stage directions (for example lights, silences etc.) are still in development

@ -1,12 +1,24 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# PLAY_ACT.py
# This script runs the play
# It is in a seperate file to enable the mechanism to detect the Google Home speaking, before continuing to the next line
# Libraries
from config import characters, directions
from logic import tts, read_script
from pixel_ring import pixel_ring
from subprocess import call
import paho.mqtt.client as mqtt
import json
import sys
from time import sleep
from logic import tts, read_script
from config import characters, directions
# Switch of LED's of speakers at the start of the play
pixel_ring.off()
# === SETUP OF MQTT PART 1 ===
@ -14,9 +26,6 @@ from config import characters, directions
HOST = 'localhost'
PORT = 1883
# === FUNCTIONS THAT ARE TRIGGERED WHEN AN INTENT IS DETECTED ===
# Subscribe to relevant MQTT topics
def on_connect(client, userdata, flags, rc):
print("Connected to {0} with result code {1}".format(HOST, rc))
@ -24,35 +33,55 @@ def on_connect(client, userdata, flags, rc):
client.subscribe("hermes/asr/textCaptured")
client.subscribe("hermes/dialogueManager/sessionQueued")
# Set Flag when Google Home is done speaking
def on_message(client, userdata, msg):
# Function which sets a flag when the Google Home is not speaking
# Callback of MQTT message that says that the text is captured by the speech recognition (ASR)
def done_speaking(client, userdata, msg):
print('Google Home is not speaking anymore')
client.connected_flag=True
# Remove any detected intents that are activated by the speaking Google Home
def on_waiting(client, userdata, msg):
# Function which removes intents that are by accident activated by the Google Home
# e.g. The google home says introduce yourself, which could trigger the other speakers to introduce themselves
# Snips works with queing of sessions, so this situation would only happen after this play is finished
def remove_sessions(client, userdata, msg):
sessionId = json.loads(id.payload)
print('delete mistaken intent')
client.publish("hermes/dialogueManager/endSession", json.dumps({
'sessionId': sessionId,
}))
# === SETUP OF MQTT PART 2 ===
# Initialise MQTT client
client = mqtt.Client()
client.connect(HOST, PORT, 60)
client.on_connect = on_connect
client.connected_flag=False
# === Read script and run the play ===
# Flags to check if the system is listening, or not
client.connected_flag=False
listening = False
# Read the script and run the play
for character, line, direction in read_script('play_scripts/interruption_02.txt'):
file = sys.argv[1] # get the chosen act passed by smart_speaker_theatre.py
for character, line, direction in read_script(file):
input_text = line
voice = characters.get(character)[0]
speaker = characters.get(character)[1]
#speaker = 'default'
# Some way to do something with the stage directions will come here
action = directions.get(direction[0])
pixel_ring.speak()
tts(voice, input_text, speaker)
if action == 'listen_google_home':
@ -67,28 +96,32 @@ for character, line, direction in read_script('play_scripts/interruption_02.txt'
# }
# }))
# Activate the microphone and speech recognition
client.publish("hermes/asr/startListening", json.dumps({
'siteId': 'default'
}))
# LED to listening mode
pixel_ring.listen()
# create callback
client.on_message = on_message
client.on_message = done_speaking
listening = True
while listening:
client.loop()
#client.on_message = on_message
client.message_callback_add('hermes/asr/textCaptured', on_message)
client.message_callback_add('hermes/asr/textCaptured', done_speaking)
if client.connected_flag:
sleep(1)
print('Continue the play')
client.connected_flag = False
client.message_callback_add('hermes/dialogueManager/sessionQueued', on_waiting)
client.message_callback_add('hermes/dialogueManager/sessionQueued', remove_sessions)
break
if action == 4:
if action == 'music':
print('play audioclip')
playing = True
@ -98,8 +131,8 @@ for character, line, direction in read_script('play_scripts/interruption_02.txt'
sleep(1)
pixel_ring.off() # Switch of the lights when done speaking
sleep(1) # Add a short pause between the lines
print('The act is over.')
print('The act is done.')

@ -1,104 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Libraries
import paho.mqtt.client as mqtt
import json
from time import sleep
from logic import tts, read_script
from config import characters, directions
# === SETUP OF MQTT PART 1 ===
# Location of the MQTT server
HOST = 'localhost'
PORT = 1883
# Subscribe to relevant MQTT topics
def on_connect(client, userdata, flags, rc):
print("Connected to {0} with result code {1}".format(HOST, rc))
client.subscribe('hermes/intent/jocavdh:play_intro_act') # to check for intent to play the act
client.subscribe('hermes/intent/jocavdh:question_continue_act') # to check for the intent to continue to the next act
client.subscribe("hermes/nlu/intentNotRecognized") # to check if the speaker didn't understand the user, trigger for fallback function
# === FUNCTIONS THAT ARE TRIGGERED WHEN AN INTENT IS DETECTED ===
# Function which is triggered when the intent play_intro_act is activated
def on_play_intro_act(client,data,msg):
data = json.loads(msg.payload)
sessionId = data['sessionId']
script_lines = read_script('play_scripts/demo.txt') # pick a random introduction script
for character, line, direction in script_lines:
input_text = line
voice = characters.get(character)[0]
speaker = characters.get(character)[1]
#speaker = 'default'
# Some way to do something with the stage directions will come here
action = directions.get(direction[0])
tts(voice, input_text, speaker)
print('say this sentence')
# if action == 'start_interrogation':
# call(["python3", "play_interrogation_act.py"])
if action == 'listen_audience':
print('listen to the audience')
client.publish('hermes/dialogueManager/endSession', json.dumps({
'sessionId': sessionId
}))
client.publish('hermes/dialogueManager/startSession', json.dumps({
'siteId': 'default',
'init': {'type': 'action', 'canBeEnqueued': True, 'intentFilter':['jocavdh:question_continue_act']}
}))
break
# Function which is triggered when the intent question_continue_act is activated
def on_question_continue_act(client,data,msg):
data = json.loads(msg.payload)
answer_value = data['slots'][0]['value']['value']
print(answer_value)
voice = "dfki-obadiah"
speaker = 'default'
if answer_value == 'no':
print('no')
if answer_value == 'yes':
call(["python3", "play_interrogation_act.py"])
print('The play is over.')
# Function which is triggered when no intent is recognized
def onIntentNotRecognized(client, data, msg):
data = json.loads(msg.payload)
line_number = 1
print('We continue on line')
print(line_number)
on_introduce(client,data,msg, line_number)
# === SETUP OF MQTT PART 2 ===
# Initialise MQTT client
client = mqtt.Client()
client.connect(HOST, PORT, 60)
client.on_connect = on_connect
# Connect each MQTT topic to which you subscribed to a handler function
client.message_callback_add('hermes/intent/jocavdh:play_intro_act', on_play_intro_act)
client.message_callback_add('hermes/intent/jocavdh:question_continue_act', on_question_continue_act)
client.message_callback_add("hermes/nlu/intentNotRecognized", onIntentNotRecognized)
# Keep checking for new MQTT messages
client.loop_forever()

@ -1,22 +1,14 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# play_config.py
# config.py
# This script contains the basic configuration of the play
# A list of the characters and their voices
# A list of stage directions which connect to formal instructions for the hardware
# ---
# Define sound output device here
# sound_output_device = sc.get_speaker('Scarlett')
# Dictionary to link characters to the right voice
characters = {"ROGUE":["dfki-prudence", "mono1"], "SAINT":["dfki-obadiah", "mono2"], "RASA":["dfki-poppy-hsmm", "mono3"] }
characters = {"ROGUE":["cmu-slt-hsmm", "mono2"], "SAINT":["dfki-obadiah-hsmm", "mono3"], "RASA":["dfki-poppy-hsmm", "mono1"] }
# Dictionary to link stage directions to a particular formal action
directions = {
'Ask audience':'listen_audience',
'Start interrogation':'start_interrogation',
'Listens to Google Home':'listen_google_home',
'Music':'audio'
}
directions = {"Listen to Google Home":'listen_google_home','Music':'music'}

@ -2,7 +2,7 @@
# -*- coding: utf-8 -*-
# play_logic.py
# logic.py
# This script contains the logic to turn the play_script into instructions for the hardware
# ---
@ -46,10 +46,19 @@ import os
from urllib.parse import urlencode, quote # For URL creation
def tts(voice, input_text, speaker):
if speaker == "mono3":
volume_level = "amount:0.5"
else:
volume_level = "amount:1.0"
# Build the query
query_hash = {"INPUT_TEXT": input_text,
"INPUT_TYPE":"TEXT", # Input text
"LOCALE":"en_GB",
"effect_VOLUME_selected":"on",
"effect_VOLUME_parameters":volume_level,
"VOICE": voice, # Voice informations (need to be compatible)
"OUTPUT_TYPE":"AUDIO",
"AUDIO":"WAVE", # Audio informations (need both)
@ -70,10 +79,8 @@ def tts(voice, input_text, speaker):
f.write(content)
f.close()
# aplay -D mono3 /tmp/output_wav.wav
#call(["aplay", "-D", "sysdefault:CARD=ArrayUAC10", fpath])
call(["aplay", fpath])
call(["aplay", "-D", speaker, "/tmp/output_wav.wav"])
else:

@ -1 +1,3 @@
SAINT: [Ask audience] do you want to continue?
ROGUE: Do you want to continue?
RASA: Well, I definitely want to
SAINT: So do I

@ -1,2 +1,2 @@
SAINT: Yes, I am ready to go. But first tell a bit more about that silly project of yours.
SAINT: [Wait for audience] Sorry, I gonna let you finish, but do you mind if I introduce myself first?
SAINT: Sorry, I gonna let you finish, but do you mind if I introduce myself first?

@ -1,3 +0,0 @@
# Needed for Snips platform 1.1.0 (0.61.1)
hermes-python>=0.3.3
toml

@ -1,4 +0,0 @@
#!/bin/bash
sudo systemctl restart snips-skill-server -v &
sudo systemctl restart snips-watch -v

@ -1,34 +0,0 @@
#!/usr/bin/env bash
set -e
# Copy config.ini.default if it exists and config.ini doesn't exist.
if [ -e config.ini.default ] && [ ! -e config.ini ]; then
cp config.ini.default config.ini
chmod a+w config.ini
fi
PYTHON=$(command -v python3)
VENV=venv
if [ -f "$PYTHON" ]; then
if [ ! -d $VENV ]; then
# Create a virtual environment if it doesn't exist.
$PYTHON -m venv $VENV
else
if [ -e $VENV/bin/python2 ]; then
# If a Python2 environment exists, delete it first
# before creating a new Python 3 virtual environment.
rm -r $VENV
$PYTHON -m venv $VENV
fi
fi
# Activate the virtual environment and install requirements.
# shellcheck disable=SC1090
. $VENV/bin/activate
pip3 install -r requirements.txt
else
>&2 echo "Cannot find Python 3. Please install it."
fi

@ -0,0 +1,83 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# SMART SPEAKER THEATRE
# This script reads the triggers for activated user intents sent by Snips over MQTT
# Using this triggers, it will start particular actions and scripts
# Libraries
import re
from config import characters, directions
from logic import tts, read_script
from subprocess import call
import paho.mqtt.client as mqtt
import json
from time import sleep
from pixel_ring import pixel_ring
# === SETUP OF MQTT PART 1 ===
# Location of the MQTT server
HOST = 'localhost'
PORT = 1883
# Subscribe to relevant MQTT topics
def on_connect(client, userdata, flags, rc):
print("Connected to {0} with result code {1}".format(HOST, rc))
client.subscribe('hermes/intent/jocavdh:play_intro_act') # to check for intent to play the act
client.subscribe('hermes/intent/jocavdh:question_continue_act') # to check for the intent to continue to the next act
client.subscribe('hermes/hotword/default/detected')
# === FUNCTIONS THAT ARE TRIGGERED WHEN AN INTENT IS DETECTED ===
def on_wakeword(client, userdata, msg):
pixel_ring.think()
# Function which is triggered when the intent play_intro_act is activated
def on_play_act(client, userdata, msg):
# # disable this intent to avoid playing another act triggered by the Google Home
# client.publish("hermes/dialogueManager/configure", json.dumps({
# 'siteId': 'default',
# 'intents': {
# 'jocavdh:play': False
# }
# }))
call(["python3", "act.py", "play_scripts/demo.txt"])
# Function which is triggered when the intent introduction is activated
def on_play_introduction(client,data,msg):
for character, line, direction in read_script('plays/introduction.txt'):
input_text = line
voice = characters.get(character)[0]
speaker = characters.get(character)[1]
action = directions.get(direction[0])
tts(voice, input_text, speaker)
sleep(1) # add a pause between each line
print('The act is over.')
# === SETUP OF MQTT PART 2 ===
# Initialise MQTT client
client = mqtt.Client()
client.connect(HOST, PORT, 60)
client.on_connect = on_connect
# Connect each MQTT topic to which you subscribed to a handler function
client.message_callback_add('hermes/intent/jocavdh:play_intro_act', on_play_act)
client.message_callback_add('hermes/hotword/default/detected', on_wakeword)
# Keep checking for new MQTT messages
client.loop_forever()

Binary file not shown.

@ -1,4 +0,0 @@
#!/bin/bash
snips-skill-server -v &
snips-watch -v

@ -1,3 +0,0 @@
#!/bin/bash
~/marytts/bin/marytts-server
Loading…
Cancel
Save