1 package org.daisy.pipeline.tts;
2
3 import java.net.URL;
4 import java.io.ByteArrayInputStream;
5 import java.io.InputStream;
6 import java.io.IOException;
7 import java.util.Collection;
8 import java.util.HashMap;
9 import java.util.List;
10 import java.util.Map;
11
12 import javax.sound.sampled.AudioFormat;
13 import javax.sound.sampled.AudioInputStream;
14 import javax.sound.sampled.UnsupportedAudioFileException;
15
16 import net.sf.saxon.Configuration;
17 import net.sf.saxon.s9api.SaxonApiException;
18 import net.sf.saxon.s9api.XdmNode;
19
20 import org.daisy.common.saxon.xslt.ThreadUnsafeXslTransformer;
21 import org.daisy.common.saxon.xslt.XslTransformCompiler;
22 import org.daisy.pipeline.audio.AudioUtils;
23 import org.daisy.pipeline.tts.TTSRegistry.TTSResource;
24 import org.daisy.pipeline.tts.TTSService.SynthesisException;
25
26 /**
27 * Classes that inherit from TTSEngine are the ones that deal with adapting
28 * external TTS processors (e.g. eSpeak and SAPI) to the Pipeline interface,
29 * which is used notably by the TextToPcm threads in the conversion from SSML to
30 * mp3. TTSEngines are meant to be allocated for every new session so that the
31 * end user can change the TTSEngine's parameters between two Pipeline jobs.
32 * Most of the methods of TTSEngine must be thread-safe.
33 */
34 public abstract class TTSEngine {
35
36 public static class SynthesisResult {
37 /**
38 * The audio produced by the TTS processor. The {@link AudioFormat} of
39 * the stream must be the same every time the same voice is used.
40 */
41 public final AudioInputStream audio;
42 /**
43 * The mark offsets (in bytes) corresponding to the
44 * <code>ssml:mark</code> elements of the input SSML. The order must be
45 * kept. The offsets are relative to the {@link AudioInputStream}. That
46 * is, they start at 0. May be <code>null</code> if the TTS processor
47 * doesn't handle SSML marks.
48 */
49 public final List<Integer> marks;
50 public SynthesisResult(AudioInputStream audio) {
51 this(audio, null);
52 }
53 public SynthesisResult(AudioInputStream audio, List<Integer> marks) {
54 this.audio = audio;
55 this.marks = marks;
56 }
57 }
58
59 /**
60 * @param provider is the service from which the TTSEngine has been
61 * allocated.
62 */
63 protected TTSEngine(TTSService provider) {
64 mProvider = provider;
65 }
66
67 /**
68 * @return the service from which the TTSEngine has been allocated.
69 */
70 public TTSService getProvider() {
71 return mProvider;
72 }
73
74 protected TTSService mProvider;
75
76 /**
77 * This method must be thread-safe. But @param threadResources is here to
78 * prevent the service from locking internal resources.
79 *
80 * @param sentence is the sentence to synthesize, as an SSML node. The node
81 * is expected to be an {@code s} element with an
82 * {@code xml:lang} attribute. It is however advised that
83 * implementations do not assume this is the case. It can
84 * also not be assumed that the node is a root element.
85 * @param voice is the voice the synthesizer must use. It is guaranteed to
86 * be one of those returned by getAvailableVoices(). This
87 * parameter can't be null.
88 * @param threadResources is the object returned by
89 * allocateThreadResource(). It may contain small persistent
90 * buffers, opened file streams, TCP connections and so on. The
91 * boolean field 'released' is guaranteed to be false, i.e. the
92 * resource provided is always valid and will remain so during
93 * the call.
94 *
95 * @return a {@link SynthesisResult} object containing the audio data and the
96 * mark offsets.
97 */
98 abstract public SynthesisResult synthesize(XdmNode sentence,
99 Voice voice,
100 TTSResource threadResources)
101 throws SynthesisException, InterruptedException;
102
103 /**
104 * Return the list of available voices for this engine. Voices that come before other voices in
105 * the list are prioritized by {@link VoiceManager} when there are no other discriminating
106 * factors.
107 *
108 * Need not be thread-safe. This method is called from the main thread.
109 */
110 abstract public Collection<Voice> getAvailableVoices() throws SynthesisException,
111 InterruptedException;
112
113 /**
114 * This method must be thread-safe. It allocates new resources (such as TCP
115 * connections) unique for each thread. Allocations can be made on-the-fly
116 * from different threads. It must not catch InterruptuedExceptions.
117 *
118 * @return the resources. Must not be null.
119 * @throws SynthesisException
120 */
121 public TTSResource allocateThreadResources() throws SynthesisException,
122 InterruptedException {
123 return new TTSResource();
124 }
125
126 /**
127 * This method must be thread-safe. Deallocations may be performed from
128 * different threads but are always performed in the same thread as the one
129 * exploiting @param resources.
130 *
131 * @param resources is the object returned by allocateThreadResource()
132 */
133 public void releaseThreadResources(TTSResource resources) throws SynthesisException,
134 InterruptedException {
135 }
136
137 /**
138 * Force interruption of the execution of synthesize() when the thread-level
139 * interruption is not enough to make synthesize() finish. Must be
140 * thread-safe, although the method must not wait for locks.
141 *
142 * @param resource is the same as the one provided to synthesize()
143 */
144 public void interruptCurrentWork(TTSResource resource) {
145 }
146
147 /**
148 * Need not be thread-safe. This method is called from the main thread.
149 */
150 public int getOverallPriority() {
151 return 1;
152 }
153
154 /**
155 * @return the number of text-to-pcm threads reserved for this TTS
156 * processor. Different values than zero make only sense if the TTS
157 * speed is limited by the number of opened resources rather than by
158 * the number of cores. In such cases, we want to maximize the time
159 * spent on using the TTS resources by avoiding using threads for
160 * something else (e.g. audio encoding or calling other TTS
161 * processors).
162 */
163 public int reservedThreadNum() {
164 return 0;
165 }
166
167 /**
168 * @return the average number of milliseconds the TTS processors is expected
169 * to spend to process a single word on a single CPU core. It
170 * doesn't need to be accurate. It is used for tuning the timeouts.
171 */
172 public int expectedMillisecPerWord() {
173 return 100;
174 }
175
176 /**
177 * @return {@code true} if the TTS engine handles SSML marks, {@code false}
178 * otherwise. Must be thread-safe.
179 */
180 public boolean handlesMarks() {
181 return false;
182 }
183
184 /**
185 * Whether the TTS engine handles
186 * <a href="https://www.w3.org/TR/speech-synthesis11/#S3.2.4"><code>prosody</code></a>
187 * elements with a {@code rate} attribute. If this method returns {@code true}, the
188 * engine is assumed to support all of the following values:
189 *
190 * <ul>
191 * <li>"x-slow", "slow", "medium", "fast", "x-fast", or "default". </li>
192 * <li>A non-negative percentage, which acts as a multiplier of the default
193 * rate.</li>
194 * <li>A number that represents speaking rate in words per minute. Note that this
195 * does not come from the SSML specification, but it is a value of the CSS
196 * <a href="https://www.w3.org/TR/CSS2/aural.html#propdef-speech-rate"><code>speech-rate</code></a>
197 * property.</li>
198 * </ul>
199 *
200 * All values, except for absolute numbers, are relative to the default rate, which
201 * is determined by the {@code org.daisy.pipeline.tts.speech-rate} property.
202 */
203 public boolean handlesSpeakingRate() {
204 return false;
205 }
206
207 /**
208 * Whether the TTS engine handles <a
209 * href="https://www.w3.org/TR/speech-synthesis11/#S3.2.4"><code>phoneme</code></a> elements,
210 * which is required for prununciation lexicons to have an effect. This method does not say
211 * anyhing about which pronunciation alphabets are supported, which phonemes, and for which
212 * languages.
213 */
214 public boolean handlesPronunciation() {
215 return false;
216 }
217
218 /* -------------------------------------------- */
219 /* HELPER FUNCTIONS */
220 /* -------------------------------------------- */
221
222 /**
223 * Transform an SSML node to a string using a given XSLT and parameter map
224 *
225 * If this method is called from {@link #synthesize}, the XSLT should not
226 * assume that the SSML node is a root element.
227 */
228 protected String transformSsmlNodeToString(XdmNode ssml, URL xslt, Map<String,Object> params)
229 throws IOException, SaxonApiException {
230 return compileXslt(xslt, ssml.getUnderlyingNode().getConfiguration())
231 .transformToString(ssml, params);
232 }
233
234 /**
235 * Compile an XSLT.
236 */
237 private ThreadUnsafeXslTransformer compileXslt(URL xslt, Configuration config)
238 throws SaxonApiException, IOException {
239 Map<URL,ThreadUnsafeXslTransformer> cache = compiledXslts.get().get(config);
240 if (cache == null) {
241 cache = new HashMap<URL,ThreadUnsafeXslTransformer>();
242 compiledXslts.get().put(config, cache);
243 }
244 ThreadUnsafeXslTransformer transformer = cache.get(xslt);
245 if (transformer == null) {
246 transformer = new XslTransformCompiler(config)
247 .compileStylesheet(xslt.openStream())
248 .newTransformer();
249 cache.put(xslt, transformer);
250 }
251 return transformer;
252 }
253
254 // Normally the same thread uses only one Configuration so
255 // ThreadLocal<Map<URL,ThreadUnsafeXslTransformer>> would also work, but we do it this way to be safe.
256 private static ThreadLocal<Map<Configuration,Map<URL,ThreadUnsafeXslTransformer>>> compiledXslts
257 = ThreadLocal.withInitial(() -> {
258 return new HashMap<Configuration,Map<URL,ThreadUnsafeXslTransformer>>(); });
259
260 /**
261 * Create an {@link AudioInputStream} from an {@link AudioFormat} and the audio data.
262 */
263 protected static AudioInputStream createAudioStream(AudioFormat format, byte[] data) {
264 return AudioUtils.createAudioStream(format, data);
265 }
266
267 protected static AudioInputStream createAudioStream(AudioFormat format, ByteArrayInputStream data) {
268 return AudioUtils.createAudioStream(format, data);
269 }
270
271 /**
272 * Create a {@link AudioInputStream} from a {@link InputStream}.
273 *
274 * This is to work around a bug in {@link javax.sound.sampled.AudioSystem}
275 * which may return {@link AudioInputStream} with a wrong {@link
276 * AudioInputStream#getFrameLength()}.
277 */
278 protected static AudioInputStream createAudioStream(InputStream stream)
279 throws UnsupportedAudioFileException, IOException {
280 return AudioUtils.createAudioStream(stream);
281 }
282 }