feat(SpeechToText): Add support for Web Sockets (#376)

ehdsouza · germanattanasio · commit 885f6800979f · 2018-03-02T17:59:16.000-05:00
* Initial implementation of set using web sockets

* Speech to text using web sockets

* fix: Adding the pyOpenSSL

* docs: Adding IBM copyright

* refactor: Following clean coding practices

better naming conventions, added method description in the method body

* refactor(Better naming conventions):

* refactor: removing the extra close signal

* refactor: naming conventions, error message

* Refactor: remove print

* feat: Adding microphone example for speech to text
diff --git a/examples/microphone-speech-to-text.py b/examples/microphone-speech-to-text.py
@@ -0,0 +1,64 @@
+from __future__ import print_function
+import pyaudio
+import tempfile
+from watson_developer_cloud import SpeechToTextV1
+from watson_developer_cloud.websocket import RecognizeCallback, RecognizeListener
+
+speech_to_text = SpeechToTextV1(
+    username='YOUR SERVICE USERNAME',
+    password='YOUR SERVICE PASSWORD',
+    url='https://stream.watsonplatform.net/speech-to-text/api')
+
+# Example using websockets
+class MyRecognizeCallback(RecognizeCallback):
+    def __init__(self):
+        pass
+
+    def on_transcription(self, transcript):
+        print(transcript)
+
+    def on_connected(self):
+        print('Connection was successful')
+
+    def on_error(self, error):
+        print('Error received: {}'.format(error))
+
+    def on_inactivity_timeout(self, error):
+        print('Inactivity timeout: {}'.format(error))
+
+    def on_listening(self):
+        print('Service is listening')
+
+    def on_transcription_complete(self):
+        print('Transcription completed')
+
+    def on_hypothesis(self, hypothesis):
+        print(hypothesis)
+
+mycallback = MyRecognizeCallback()
+tmp = tempfile.NamedTemporaryFile()
+
+FORMAT = pyaudio.paInt16
+CHANNELS = 1
+RATE = 44100
+CHUNK = 1024
+RECORD_SECONDS = 5
+
+audio = pyaudio.PyAudio()
+stream = audio.open(format=FORMAT, channels=CHANNELS,
+                rate=RATE, input=True,
+                frames_per_buffer=CHUNK)
+
+print('recording....')
+with open(tmp.name, 'w') as f:
+    for i in range(0, int(RATE / CHUNK * RECORD_SECONDS)):
+        data = stream.read(CHUNK)
+        f.write(data)
+
+stream.stop_stream()
+stream.close()
+audio.terminate()
+print('Done recording...')
+
+with open(tmp.name) as f:
+    speech_to_text.recognize_with_websocket(audio=f, recognize_callback=mycallback)
diff --git a/examples/speech_to_text_v1.py b/examples/speech_to_text_v1.py
@@ -2,10 +2,12 @@
 import json
 from os.path import join, dirname
 from watson_developer_cloud import SpeechToTextV1
+from watson_developer_cloud.websocket import RecognizeCallback, RecognizeListener
 
 speech_to_text = SpeechToTextV1(
     username='YOUR SERVICE USERNAME',
-    password='YOUR SERVICE PASSWORD')
+    password='YOUR SERVICE PASSWORD',
+    url='https://stream.watsonplatform.net/speech-to-text/api')
 
 print(json.dumps(speech_to_text.list_models(), indent=2))
 
@@ -21,3 +23,35 @@
                 timestamps=True,
                 word_confidence=True),
             indent=2))
+
+# Example using websockets
+class MyRecognizeCallback(RecognizeCallback):
+    def __init__(self):
+        pass
+
+    def on_transcription(self, transcript):
+        print(transcript)
+
+    def on_connected(self):
+        print('Connection was successful')
+
+    def on_error(self, error):
+        print('Error received: {}'.format(error))
+
+    def on_inactivity_timeout(self, error):
+        print('Inactivity timeout: {}'.format(error))
+
+    def on_listening(self):
+        print('Service is listening')
+
+    def on_transcription_complete(self):
+        print('Transcription completed')
+
+    def on_hypothesis(self, hypothesis):
+        print(hypothesis)
+
+mycallback = MyRecognizeCallback()
+with open(join(dirname(__file__), '../resources/speech.wav'),
+          'rb') as audio_file:
+    speech_to_text.recognize_with_websocket(
+        audio=audio_file, recognize_callback=mycallback)
diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -15,3 +15,9 @@ pytest-cov>=2.2.1
 recommonmark>=0.2.0
 Sphinx>=1.3.1
 bumpversion>=0.5.3
+
+# Web sockets
+autobahn>=0.10.9
+Twisted>=13.2.0
+pyOpenSSL>=16.2.0
+service-identity>=17.0.0
diff --git a/requirements.txt b/requirements.txt
@@ -1,2 +1,6 @@
 requests>=2.0,<3.0
 python_dateutil>=2.5.3
+autobahn>=0.10.9
+Twisted>=13.2.0
+pyOpenSSL>=16.2.0
+service-identity>=17.0.0
diff --git a/test/integration/test_examples.py b/test/integration/test_examples.py
@@ -10,7 +10,7 @@
 from glob import glob
 
 # tests to exclude
-excludes = ['authorization_v1.py', 'discovery_v1.ipynb', '__init__.py']
+excludes = ['authorization_v1.py', 'discovery_v1.ipynb', '__init__.py', 'microphone-speech-to-text.py']
 
 # examples path. /examples
 examples_path = join(dirname(__file__), '../', 'examples', '*.py')
diff --git a/watson_developer_cloud/speech_to_text_v1.py b/watson_developer_cloud/speech_to_text_v1.py
@@ -25,9 +25,12 @@
 from __future__ import absolute_import
 
 import json
-from .watson_service import WatsonService
+from .watson_service import WatsonService, _remove_null_values
 from .utils import deprecated
-
+from watson_developer_cloud.websocket import RecognizeCallback, RecognizeListener
+from os.path import isfile
+import base64
+import urllib
 ##############################################################################
 # Service
 ##############################################################################
@@ -183,6 +186,96 @@ def recognize(self,
             accept_json=True)
         return response
 
+    def recognize_with_websocket(self,
+                                audio=None,
+                                content_type='audio/l16; rate=44100',
+                                model='en-US_BroadbandModel',
+                                recognize_callback=None,
+                                customization_id=None,
+                                acoustic_customization_id=None,
+                                customization_weight=None,
+                                version=None,
+                                inactivity_timeout=None,
+                                interim_results=True,
+                                keywords=None,
+                                keywords_threshold=None,
+                                max_alternatives=1,
+                                word_alternatives_threshold=None,
+                                word_confidence=False,
+                                timestamps=False,
+                                profanity_filter=None,
+                                smart_formatting=False,
+                                speaker_labels=None):
+        """
+        Sends audio for speech recognition using web sockets.
+
+        :param str audio: Audio to transcribe in the format specified by the `Content-Type` header.
+        :param str content_type: The type of the input: audio/basic, audio/flac, audio/l16, audio/mp3, audio/mpeg, audio/mulaw, audio/ogg, audio/ogg;codecs=opus, audio/ogg;codecs=vorbis, audio/wav, audio/webm, audio/webm;codecs=opus, audio/webm;codecs=vorbis, or multipart/form-data.
+        :param str model: The identifier of the model to be used for the recognition request.
+        :param RecognizeCallback recognize_callback: The instance handling events returned from the service.
+        :param str customization_id: The GUID of a custom language model that is to be used with the request. The base model of the specified custom language model must match the model specified with the `model` parameter. You must make the request with service credentials created for the instance of the service that owns the custom model. By default, no custom language model is used.
+        :param str acoustic_customization_id: The GUID of a custom acoustic model that is to be used with the request. The base model of the specified custom acoustic model must match the model specified with the `model` parameter. You must make the request with service credentials created for the instance of the service that owns the custom model. By default, no custom acoustic model is used.
+        :param float customization_weight: If you specify a `customization_id` with the request, you can use the `customization_weight` parameter to tell the service how much weight to give to words from the custom language model compared to those from the base model for speech recognition.   Specify a value between 0.0 and 1.0. Unless a different customization weight was specified for the custom model when it was trained, the default value is 0.3. A customization weight that you specify overrides a weight that was specified when the custom model was trained.   The default value yields the best performance in general. Assign a higher value if your audio makes frequent use of OOV words from the custom model. Use caution when setting the weight: a higher value can improve the accuracy of phrases from the custom model's domain, but it can negatively affect performance on non-domain phrases.
+        :param str version: The version of the specified base `model` that is to be used for speech recognition. Multiple versions of a base model can exist when a model is updated for internal improvements. The parameter is intended primarily for use with custom models that have been upgraded for a new base model. The default value depends on whether the parameter is used with or without a custom model. For more information, see [Base model version](https://console.bluemix.net/docs/services/speech-to-text/input.html#version).
+        :param int inactivity_timeout: The time in seconds after which, if only silence (no speech) is detected in submitted audio, the connection is closed with a 400 error. Useful for stopping audio submission from a live microphone when a user simply walks away. Use `-1` for infinity.
+        :param bool interim_results: Send back non-final previews of each "sentence" as it is being processed. These results are ignored in text mode.
+        :param list[str] keywords: Array of keyword strings to spot in the audio. Each keyword string can include one or more tokens. Keywords are spotted only in the final hypothesis, not in interim results. If you specify any keywords, you must also specify a keywords threshold. Omit the parameter or specify an empty array if you do not need to spot keywords.
+        :param float keywords_threshold: Confidence value that is the lower bound for spotting a keyword. A word is considered to match a keyword if its confidence is greater than or equal to the threshold. Specify a probability between 0 and 1 inclusive. No keyword spotting is performed if you omit the parameter. If you specify a threshold, you must also specify one or more keywords.
+        :param int max_alternatives: Maximum number of alternative transcripts to be returned. By default, a single transcription is returned.
+        :param float word_alternatives_threshold: Confidence value that is the lower bound for identifying a hypothesis as a possible word alternative (also known as \"Confusion Networks\"). An alternative word is considered if its confidence is greater than or equal to the threshold. Specify a probability between 0 and 1 inclusive. No alternative words are computed if you omit the parameter.
+        :param bool word_confidence: If `true`, confidence measure per word is returned.
+        :param bool timestamps: If `true`, time alignment for each word is returned.
+        :param bool profanity_filter: If `true` (the default), filters profanity from all output except for keyword results by replacing inappropriate words with a series of asterisks. Set the parameter to `false` to return results with no censoring. Applies to US English transcription only.
+        :param bool smart_formatting: If `true`, converts dates, times, series of digits and numbers, phone numbers, currency values, and Internet addresses into more readable, conventional representations in the final transcript of a recognition request. If `false` (the default), no formatting is performed. Applies to US English transcription only.
+        :param bool speaker_labels: Indicates whether labels that identify which words were spoken by which participants in a multi-person exchange are to be included in the response. The default is `false`; no speaker labels are returned. Setting `speaker_labels` to `true` forces the `timestamps` parameter to be `true`, regardless of whether you specify `false` for the parameter.   To determine whether a language model supports speaker labels, use the `GET /v1/models` method and check that the attribute `speaker_labels` is set to `true`. You can also refer to [Speaker labels](https://console.bluemix.net/docs/services/speech-to-text/output.html#speaker_labels).
+        :return:
+        """
+        if audio is None:
+            raise ValueError('Audio must be provided')
+        if recognize_callback is None:
+            raise ValueError('Recognize callback must be provided')
+        if not isinstance(recognize_callback, RecognizeCallback):
+            raise Exception('Callback is not a derived class of RecognizeCallback')
+
+        headers = {}
+        if self.default_headers is not None:
+            headers = self.default_headers.copy()
+        base64_authorization = base64.b64encode(self.username +  ':' + self.password)
+        headers['Authorization'] = 'Basic {0}'.format(base64_authorization)
+
+        url = self.url.replace('https:', 'wss:')
+        params = {
+                    'model': model,
+                    'customization_id': customization_id,
+                    'acoustic_customization_id': acoustic_customization_id,
+                    'customization_weight': customization_weight,
+                    'version': version
+                 }
+        params = _remove_null_values(params)
+        url = url + '/v1/recognize?{0}'.format(urllib.urlencode(params))
+
+        options = {
+                    'content_type': content_type,
+                    'inactivity_timeout': inactivity_timeout,
+                    'interim_results': interim_results,
+                    'keywords': keywords,
+                    'keywords_threshold': keywords_threshold,
+                    'max_alternatives': max_alternatives,
+                    'word_alternatives_threshold': word_alternatives_threshold,
+                    'word_confidence': word_confidence,
+                    'timestamps': timestamps,
+                    'profanity_filter': profanity_filter,
+                    'smart_formatting': smart_formatting,
+                    'speaker_labels': speaker_labels
+                  }
+        options = _remove_null_values(options)
+
+        recognizeListener = RecognizeListener(audio,
+                                              options,
+                                              recognize_callback,
+                                              url,
+                                              headers)
+
     #########################
     # asynchronous
     #########################
diff --git a/watson_developer_cloud/websocket/__init__.py b/watson_developer_cloud/websocket/__init__.py
@@ -0,0 +1,18 @@
+# coding: utf-8
+
+# Copyright 2018 IBM All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .recognize_abstract_callback import RecognizeCallback
+from .speech_to_text_websocket_listener import RecognizeListener
diff --git a/watson_developer_cloud/websocket/recognize_abstract_callback.py b/watson_developer_cloud/websocket/recognize_abstract_callback.py
@@ -0,0 +1,59 @@
+# coding: utf-8
+
+# Copyright 2018 IBM All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class RecognizeCallback:
+
+  def on_transcription(self, transcript):
+    """
+    Called when an interim result is received
+    """
+    pass
+
+  def on_connected(self):
+    """
+    Called when a WebSocket connection was made
+    """
+    pass
+
+  def on_error(self, error):
+    """
+    Called when there is an error in the Web Socket connection.
+    """
+    pass
+
+  def on_inactivity_timeout(self):
+    """
+    Called when there is an inactivity timeout.
+    """
+    pass
+
+  def on_listening(self):
+    """
+    Called when the service is listening for audio.
+    """
+    pass
+
+  def on_transcription_complete(self):
+    """
+    Called after the service returns the final result for the transcription.
+    """
+    pass
+
+  def on_hypothesis(self, hypothesis):
+    """
+    Called when the service returns the final hypothesis
+    """
+    pass
diff --git a/watson_developer_cloud/websocket/speech_to_text_websocket_listener.py b/watson_developer_cloud/websocket/speech_to_text_websocket_listener.py
