LT-22314: Create new media line for interlinear display (#720)

* Create new media line for interlinear display

Includes support for playing audio file segments

This supports the following workflows:
1. Clicking the play button plays the segment.
2. Clicking the same play button, while the segment is still
playing, will stop playing.
3. Clicking a different play button, while a segment is still
playing, will stop playing the current segment and start playing
the new one.
4. While the audio is playing the user can continue to work in
the gui. If the user does something that disposes the
InterlinDocForAnalysis, then the audio will stop.

Displays begin and end time offsets and speaker name
for each audio clip.

Handles bidi and display in multiple writing systems.

Displays a no media message instead of the audio play
button/info if there is no media file.

---------

Co-authored-by: mark-sil <83427558+mark-sil@users.noreply.github.com>
(cherry picked from commit cb8af5f)
(cherry picked from commit e48e9a3)

Author: 2 people

Src/FwResources/Images/InterlinPlayArrow.bmp

@@ -566,6 +566,16 @@ public static Image InterlinPopupArrow
 			get { return Helper.m_imgLst11x12.Images[0]; }
 		}
 
+		/// ------------------------------------------------------------------------------------
+		/// <summary>
+		/// Gets the interlinear audio play arrow.
+		/// </summary>
+		/// ------------------------------------------------------------------------------------
+		public static Image InterlinPlayArrow
+		{
+			get { return Helper.m_imgLst11x12.Images[1]; }
+		}
+
 		/// <summary>
 		/// Icon for linking words into phrases.
 		/// </summary>

Src/FwResources/ResourceHelper.cs

@@ -30,6 +30,7 @@
     <PackageReference Include="SIL.LCModel.Core" GeneratePathProperty="true" />
     <PackageReference Include="SIL.LCModel.Utils" />
     <PackageReference Include="SIL.Machine" />
+    <PackageReference Include="SIL.Media" />
     <PackageReference Include="SIL.ParatextShared" />
     <PackageReference Include="SIL.Windows.Forms" />
     <PackageReference Include="SIL.Windows.Forms.GeckoBrowserAdapter" />
@@ -40,6 +41,7 @@
     <PackageReference Include="AdamsLair.TreeViewAdv" />
     <PackageReference Include="SIL.ExCSS" />
     <PackageReference Include="Geckofx60.64" />
+    <PackageReference Include="NAudio" />
   </ItemGroup>
   <ItemGroup>
     <Reference Include="Accessibility" />

Src/FwResources/ResourceHelperImpl.Designer.cs

@@ -384,11 +384,13 @@ public void ConfigurationLineOptions()
 			choices.Add(InterlinLineChoices.kflidLitTrans, wsEng, true); //7
 			choices.Add(InterlinLineChoices.kflidFreeTrans, wsEng, true); //8
 
-			choices.Add(InterlinLineChoices.kflidLexGloss, wsFrn, true);
-			choices.Add(InterlinLineChoices.kflidLexGloss, wsGer, true);
+			choices.Add(InterlinLineChoices.kflidLexGloss, wsFrn, true); //9
+			choices.Add(InterlinLineChoices.kflidLexGloss, wsGer, true); //10
+
+			choices.Add(InterlinLineChoices.kflidMedia, wsEng, true); //11
 
 			// Pre-checks
-			Assert.That(choices.AllLineSpecs.Count, Is.EqualTo(11));
+			Assert.That(choices.AllLineSpecs.Count, Is.EqualTo(12));
 			Assert.That(choices.AllLineSpecs[0].Flid, Is.EqualTo(InterlinLineChoices.kflidWord));
 			Assert.That(choices.AllLineSpecs[1].Flid, Is.EqualTo(InterlinLineChoices.kflidMorphemes));
 			Assert.That(choices.AllLineSpecs[2].Flid, Is.EqualTo(InterlinLineChoices.kflidLexEntries));
@@ -400,6 +402,7 @@ public void ConfigurationLineOptions()
 			Assert.That(choices.AllLineSpecs[8].Flid, Is.EqualTo(InterlinLineChoices.kflidWordPos));
 			Assert.That(choices.AllLineSpecs[9].Flid, Is.EqualTo(InterlinLineChoices.kflidLitTrans));
 			Assert.That(choices.AllLineSpecs[10].Flid, Is.EqualTo(InterlinLineChoices.kflidFreeTrans));
+			Assert.That(choices.AllLineSpecs[11].Flid, Is.EqualTo(InterlinLineChoices.kflidMedia));
 
 			Assert.That(choices.AllLineSpecs[3].WritingSystem, Is.EqualTo(wsEng));
 			Assert.That(choices.AllLineSpecs[4].WritingSystem, Is.EqualTo(wsFrn));
@@ -408,7 +411,7 @@ public void ConfigurationLineOptions()
 			ReadOnlyCollection<LineOption> configLineOptions = choices.ConfigurationLineOptions;
 
 			// Post-checks
-			Assert.That(configLineOptions.Count, Is.EqualTo(10)); // 9 + 1 for kflidNote.
+			Assert.That(configLineOptions.Count, Is.EqualTo(11)); // 10 + 1 for kflidNote.
 			Assert.That(configLineOptions[0].Flid, Is.EqualTo(InterlinLineChoices.kflidWord));
 			Assert.That(configLineOptions[1].Flid, Is.EqualTo(InterlinLineChoices.kflidMorphemes));
 			Assert.That(configLineOptions[2].Flid, Is.EqualTo(InterlinLineChoices.kflidLexEntries));
@@ -418,8 +421,11 @@ public void ConfigurationLineOptions()
 			Assert.That(configLineOptions[6].Flid, Is.EqualTo(InterlinLineChoices.kflidWordPos));
 			Assert.That(configLineOptions[7].Flid, Is.EqualTo(InterlinLineChoices.kflidLitTrans));
 			Assert.That(configLineOptions[8].Flid, Is.EqualTo(InterlinLineChoices.kflidFreeTrans));
+			Assert.That(configLineOptions[9].Flid, Is.EqualTo(InterlinLineChoices.kflidMedia));
 			// kflidNote is one of the required options so it was added.
-			Assert.That(configLineOptions[9].Flid, Is.EqualTo(InterlinLineChoices.kflidNote));
+			// Note: kflidNote is always added as the last option, so if new line choices are added in future,
+			// kflidNote will need to be updated to match the new last index.
+			Assert.AreEqual(InterlinLineChoices.kflidNote, configLineOptions[10].Flid);
 		}
 
 		[Test]

Src/FwResources/ResourceHelperImpl.resx

@@ -341,6 +341,25 @@
     <value>Nt </value>
     <comment>short for Note; there is a trailing space</comment>
   </data>
+  <data name="ksMedia" xml:space="preserve">
+    <value>Media</value>
+  </data>
+  <data name="ksMedia_" xml:space="preserve">
+    <value>Media </value>
+    <comment>there is a trailing space</comment>
+  </data>
+  <data name="ksNoMedia" xml:space="preserve">
+    <value>No Media</value>
+  </data>
+  <data name="ksBeginTimeOffset" xml:space="preserve">
+    <value>Begin</value>
+  </data>
+  <data name="ksEndTimeOffset" xml:space="preserve">
+    <value>End</value>
+  </data>
+  <data name="ksSpeaker" xml:space="preserve">
+    <value>Speaker</value>
+  </data>
   <data name="ksPhaseButton" xml:space="preserve">
     <value>Ph{0} {1}</value>
     <comment>{0} is phase number; {1} is either "Process" or "Import"</comment>

Src/LexText/Interlinear/ITextDll.csproj

@@ -2,10 +2,12 @@
 // This software is licensed under the LGPL, version 2.1 or later
 // (http://www.gnu.org/licenses/lgpl-2.1.html)
 
+using NAudio.Wave;
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Drawing;
+using System.IO;
 using System.Linq;
 using System.Runtime.InteropServices;
 using System.Windows.Forms;
@@ -19,13 +21,20 @@
 using SIL.LCModel.DomainServices;
 using SIL.LCModel.Infrastructure;
 using SIL.LCModel.Utils;
+using SIL.Media.Naudio;
 using SIL.PlatformUtilities;
 using XCore;
 
 namespace SIL.FieldWorks.IText
 {
 	public partial class InterlinDocForAnalysis : InterlinDocRootSiteBase
 	{
+		private int m_playingSegmentHvo = -1;
+		private int m_queuedSegmentHvo = -1;
+		private WaveOutEvent m_outputDevice;
+		private AudioFileReader m_audioFile;
+		private TrimWaveStream m_trimWaveStream;
+
 		/// <summary>
 		/// Review(EricP) consider making a subclass of InterlinDocForAnalysis (i.e. InterlinDocForGlossing)
 		/// so we can put all AddWordsToLexicon related code there rather than having this
@@ -2110,7 +2119,45 @@ protected bool HandleClickSelection(IVwSelection vwselNew, bool fBundleOnly, boo
 				ScrollSelectionIntoView(this.RootBox.Selection, VwScrollSelOpts.kssoDefault);
 				return true;
 			}
+			// Play or stop playing a media segment.
+			else if (tagTextProp == SegmentTags.kflidMediaURI)
+			{
+				int itagSeg = -1;
+				for (int i = rgvsli.Length; --i >= 0;)
+				{
+					if (rgvsli[i].tag == StTxtParaTags.kflidSegments)
+					{
+						itagSeg = i;
+						break;
+					}
+				}
+				Debug.Assert(itagSeg >= 0);
+				if (itagSeg >= 0)
+				{
+					int hvoSegment = rgvsli[itagSeg].hvo;
 
+					// Nothing is currently playing, so play the segment.
+					if (m_playingSegmentHvo == -1)
+					{
+						StartMediaPlay(hvoSegment);
+					}
+					// Something is currently playing. If it's the same segment, stop it. If it's a
+					// different segment, queue up the new one and stop the current one.
+					else
+					{
+						// If 'Play' was pressed for a different segment, then queue up the new
+						// segment to play immediately after stopping the current one.
+						if (m_playingSegmentHvo != hvoSegment)
+						{
+							m_queuedSegmentHvo = hvoSegment;
+						}
+
+						// Stop the currently playing segment.
+						StopMediaPlay();
+					}
+				}
+				return true;
+			}
 			// Identify the analysis, and the position in m_rgvsli of the property holding it.
 			// It is also possible that the analysis is the root object.
 			// This is important because although we are currently displaying just an StTxtPara,
@@ -2164,6 +2211,74 @@ protected bool HandleClickSelection(IVwSelection vwselNew, bool fBundleOnly, boo
 			return true;
 		}
 
+		/// <summary>
+		/// Begins playback of the audio segment.
+		/// </summary>
+		/// <remarks>If playback is already in progress, this method should not be called until the previous
+		/// playback has stopped.</remarks>
+		/// <param name="hvoSegment">The hvo of the segment to play.</param>
+		private void StartMediaPlay(int hvoSegment)
+		{
+			Debug.Assert(m_outputDevice == null && m_audioFile == null && m_trimWaveStream == null &&
+						 m_playingSegmentHvo == -1);
+
+			var segment = Cache.ServiceLocator.GetObject(hvoSegment) as ISegment;
+			Debug.Assert(segment != null);
+
+			var mediaUri = segment.MediaURIRA?.MediaURI;
+			Uri uri = new Uri(mediaUri);
+			if (uri.IsFile && File.Exists(uri.LocalPath))
+			{
+				m_outputDevice = new WaveOutEvent();
+				m_outputDevice.PlaybackStopped += OnPlaybackStopped;
+				m_audioFile = new AudioFileReader(uri.LocalPath);
+				m_trimWaveStream = new TrimWaveStream(m_audioFile);
+
+				double beginMs = double.Parse(segment.BeginTimeOffset);
+				double endMs = double.Parse(segment.EndTimeOffset);
+				m_trimWaveStream.StartPosition = TimeSpan.FromMilliseconds(beginMs);
+				m_trimWaveStream.EndPosition = TimeSpan.FromMilliseconds(endMs);
+				m_outputDevice.Init(m_trimWaveStream);
+
+				m_outputDevice.Play();
+				m_playingSegmentHvo = hvoSegment;
+			}
+		}
+
+		/// <summary>
+		/// Stops media play, if it is currently playing. This will trigger the PlaybackStopped event,
+		/// which will clean up the audio objects and start playing any queued segment.
+		/// </summary>
+		private void StopMediaPlay()
+		{
+			m_outputDevice?.Stop();
+		}
+
+		/// <summary>
+		/// Handles the event that occurs when audio playback has stopped. After disposing of the
+		/// audio objects, if there is a queued segment to play, starts playing it.
+		/// </summary>
+		private void OnPlaybackStopped(object sender, StoppedEventArgs args)
+		{
+			m_outputDevice.Dispose();
+			m_outputDevice = null;
+			m_trimWaveStream.Dispose();
+			m_trimWaveStream = null;
+			m_audioFile.Dispose();
+			m_audioFile = null;
+			m_playingSegmentHvo = -1;
+
+			if(m_queuedSegmentHvo != -1)
+			{
+				int segmentToPlay = m_queuedSegmentHvo;
+				m_queuedSegmentHvo = -1;
+				StartMediaPlay(segmentToPlay);
+			}
+		}
+
+
+
+
 		#endregion
 
 		#region IxCoreColleague