1 package org.daisy.pipeline.tts.calabash.impl;
2
3 import java.io.File;
4 import java.io.PrintWriter;
5 import java.io.StringWriter;
6 import java.net.URI;
7 import java.net.URISyntaxException;
8 import java.time.Duration;
9 import java.util.Collections;
10 import java.util.HashMap;
11 import java.util.Map;
12 import java.util.Random;
13 import java.util.concurrent.Semaphore;
14
15 import javax.sound.sampled.AudioFileFormat;
16
17 import com.google.common.collect.Iterables;
18 import com.xmlcalabash.core.XProcException;
19 import com.xmlcalabash.core.XProcRuntime;
20 import com.xmlcalabash.io.ReadablePipe;
21 import com.xmlcalabash.io.WritablePipe;
22 import com.xmlcalabash.library.DefaultStep;
23 import com.xmlcalabash.model.RuntimeValue;
24 import com.xmlcalabash.runtime.XAtomicStep;
25 import com.xmlcalabash.util.TreeWriter;
26
27 import net.sf.saxon.s9api.QName;
28 import net.sf.saxon.s9api.SaxonApiException;
29
30 import org.daisy.common.properties.Properties;
31 import org.daisy.common.properties.Properties.Property;
32 import org.daisy.common.xproc.calabash.XProcStep;
33 import org.daisy.pipeline.audio.AudioClip;
34 import org.daisy.pipeline.audio.AudioFileTypes;
35 import org.daisy.pipeline.audio.AudioServices;
36 import org.daisy.pipeline.tts.AudioFootprintMonitor;
37 import org.daisy.pipeline.tts.calabash.impl.EncodingThread.EncodingException;
38 import static org.daisy.pipeline.tts.calabash.impl.SynthesizeProvider.ENABLE_LOG;
39 import org.daisy.pipeline.tts.config.ConfigReader;
40 import org.daisy.pipeline.tts.config.DynamicPropertiesExtension;
41 import org.daisy.pipeline.tts.config.VoiceConfigExtension;
42 import org.daisy.pipeline.tts.TTSLog;
43 import org.daisy.pipeline.tts.TTSRegistry;
44 import org.daisy.pipeline.tts.TTSLog.ErrorCode;
45 import org.daisy.pipeline.tts.TTSService.SynthesisException;
46
47 import org.slf4j.Logger;
48 import org.slf4j.LoggerFactory;
49
50 public class SynthesizeStep extends DefaultStep implements FormatSpecifications, XProcStep {
51
52 private static final Logger logger = LoggerFactory.getLogger(SynthesizeStep.class);
53 private static final Property AUDIO_TMPDIR = Properties.getProperty("org.daisy.pipeline.tts.audio.tmpdir",
54 false,
55 "Temporary directory used during speech synthesis",
56 false,
57 null);
58 private static final Property SENTENCES_PER_FILE = Properties.getProperty("org.daisy.pipeline.tts.sentences.per.file",
59 true,
60 "Maximum number of sentences per audio file",
61 false,
62 null);
63
64
65
66
67 private static QName ERR_TTS_001 = new QName("pe", "http://www.daisy.org/ns/pipeline/errors", "TTS001");
68
69 private ReadablePipe source = null;
70 private ReadablePipe config = null;
71 private WritablePipe result = null;
72 private WritablePipe status = null;
73 private WritablePipe logOutput = null;
74 private XProcRuntime mRuntime;
75 private TTSRegistry mTTSRegistry;
76 private Random mRandGenerator;
77 private AudioServices mAudioServices;
78 private Semaphore mStartSemaphore;
79 private AudioFootprintMonitor mAudioFootprintMonitor;
80 private String mTempDirOpt;
81 private boolean mIncludeLogOpt;
82 private AudioFileFormat.Type mAudioFileType;
83 private final Map<String,String> properties;
84
85 private static String convertDurationToString(Duration duration) {
86 long hours = duration.toHours();
87 long minutes = duration.toMinutes() % 60;
88 long seconds = duration.getSeconds() % 60;
89 long milliseconds = duration.getNano() / 1000000;
90 return String.format("%d:%02d:%02d.%03d",
91 hours,
92 minutes,
93 seconds,
94 milliseconds);
95 }
96
97 public SynthesizeStep(XProcRuntime runtime, XAtomicStep step, TTSRegistry ttsRegistry,
98 AudioServices audioServices, Semaphore startSemaphore,
99 AudioFootprintMonitor audioFootprintMonitor, Map<String,String> properties) {
100 super(runtime, step);
101 mStartSemaphore = startSemaphore;
102 mAudioFootprintMonitor = audioFootprintMonitor;
103 mAudioServices = audioServices;
104 mRuntime = runtime;
105 mTTSRegistry = ttsRegistry;
106 mRandGenerator = new Random();
107 this.properties = properties;
108 }
109
110 public void setInput(String port, ReadablePipe pipe) {
111 if ("source".equals(port)) {
112 source = pipe;
113 } else if ("config".equals(port)) {
114 config = pipe;
115 }
116 }
117
118 public void setOutput(String port, WritablePipe pipe) {
119 if ("result".equals(port)) {
120 result = pipe;
121 } else if ("status".equals(port)) {
122 status = pipe;
123 } else if ("log".equals(port)) {
124 logOutput = pipe;
125 }
126 }
127
128 @Override
129 public void setOption(QName name, RuntimeValue value) {
130 if ("temp-dir".equals(name.getLocalName())) {
131 mTempDirOpt = value.getString();
132 } else if ("include-log".equals(name.getLocalName())) {
133 mIncludeLogOpt = value.getBoolean();
134 } else if ("audio-file-type".equals(name.getLocalName())) {
135 mAudioFileType = AudioFileTypes.fromMediaType(value.getString());
136 if (mAudioFileType == null) {
137 logger.warn("Audio file type not recognized or not supported: " + value.getString()
138 + ". Falling back to MP3.");
139 mAudioFileType = AudioFileTypes.MP3;
140 }
141 } else
142 super.setOption(name, value);
143 }
144
145 public void reset() {
146 source.resetReader();
147 config.resetReader();
148 result.resetWriter();
149 status.resetWriter();
150 logOutput.resetWriter();
151 }
152
153 public void run() throws SaxonApiException {
154 super.run();
155
156 try {
157 mStartSemaphore.acquire();
158 } catch (InterruptedException e) {
159 logger.error("Interrupted", e);
160 return;
161 }
162
163 VoiceConfigExtension configExt = new VoiceConfigExtension();
164 DynamicPropertiesExtension propsExt = new DynamicPropertiesExtension();
165 new ConfigReader(mRuntime.getProcessor(), config.read(), this.properties, configExt, propsExt);
166 Map<String,String> properties = this.properties;
167 Map<String,String> dynProperties = propsExt.getDynamicProperties();
168 if (dynProperties != null && !dynProperties.isEmpty()) {
169 properties = new HashMap<>(properties);
170 properties.putAll(dynProperties);
171 }
172
173 boolean logEnabled = mIncludeLogOpt;
174 if (!logEnabled) {
175 String logEnabledProp = ENABLE_LOG.getValue(properties);
176
177
178
179 logEnabled = "true".equalsIgnoreCase(logEnabledProp);
180 }
181 TTSLog log = new TTSLog(logger);
182 File audioOutputDir; {
183 if (mTempDirOpt != null && !mTempDirOpt.isEmpty()) {
184 try {
185 audioOutputDir = new File(new URI(mTempDirOpt));
186 } catch (URISyntaxException|IllegalArgumentException e) {
187 throw new RuntimeException("temp-dir option invalid: " + mTempDirOpt);
188 }
189 if (audioOutputDir.exists())
190 throw new RuntimeException("temp-dir option must be a non-existing directory: "+mTempDirOpt);
191 } else {
192 String tmpDir = AUDIO_TMPDIR.getValue(properties);
193 if (tmpDir == null)
194 tmpDir = System.getProperty("java.io.tmpdir");
195 do {
196 String audioDir = tmpDir + "/";
197 for (int k = 0; k < 2; ++k)
198 audioDir += Long.toString(mRandGenerator.nextLong(), 32);
199 audioOutputDir = new File(audioDir);
200 } while (audioOutputDir.exists());
201 }
202 }
203 audioOutputDir.mkdirs();
204 audioOutputDir.deleteOnExit();
205 int maxSentencesPerSection = 100; {
206 String v = SENTENCES_PER_FILE.getValue(properties);
207 if (v != null) {
208 try {
209 maxSentencesPerSection = Integer.valueOf(v);
210 } catch (NumberFormatException e) {
211 throw new RuntimeException(v + " is not a valid a value for property " + SENTENCES_PER_FILE.getName());
212 }
213 }
214 }
215
216 SSMLtoAudio ssmltoaudio = new SSMLtoAudio(audioOutputDir, mAudioFileType, maxSentencesPerSection,
217 mTTSRegistry, logger, mAudioFootprintMonitor, mRuntime.getProcessor(), properties,
218 configExt, log);
219
220 Iterable<SoundFileLink> soundFragments = Collections.EMPTY_LIST;
221 try {
222 while (source.moreDocuments()) {
223 ssmltoaudio.feedSSML(source.read());
224 }
225 Iterable<SoundFileLink> newfrags = ssmltoaudio.blockingRun(mAudioServices);
226 soundFragments = Iterables.concat(soundFragments, newfrags);
227 } catch (SynthesisException e) {
228 logger.error("Synthesis failed", e);
229 return;
230 } catch (EncodingException e) {
231 throw new XProcException(ERR_TTS_001,
232 step,
233 "Encoding error",
234 XProcException.fromException(e)
235 .rebase(step.getLocation(),
236 new RuntimeException().getStackTrace()));
237 } catch (InterruptedException e) {
238 logger.error("Interrupted", e);
239 return;
240 } finally {
241 mStartSemaphore.release();
242 }
243
244 TreeWriter tw = new TreeWriter(runtime);
245 tw.startDocument(runtime.getStaticBaseURI());
246 tw.addStartElement(OutputRootTag);
247
248 int num = 0;
249 for (SoundFileLink link : soundFragments) {
250 AudioClip clip = link.getAudioFragment(log);
251 if (clip != null) {
252 if (!clip.clipEnd.equals(clip.clipBegin)) {
253
254 tw.addStartElement(ClipTag);
255 tw.addAttribute(Audio_attr_textref, link.getTextFragment().toString());
256 tw.addAttribute(Audio_attr_clipBegin,
257 convertDurationToString(clip.clipBegin));
258 tw.addAttribute(Audio_attr_clipEnd,
259 convertDurationToString(clip.clipEnd));
260 tw.addAttribute(Audio_attr_src, clip.src.toString());
261 tw.addEndElement();
262 }
263 ++num;
264 TTSLog.Entry entry = log.getOrCreateEntry(link.getTextFragment().toString());
265 entry.setClip(clip);
266 } else {
267 log.getOrCreateEntry(link.getTextFragment().toString()).addError(
268 new TTSLog.Error(ErrorCode.AUDIO_MISSING,
269 "not synthesized or not encoded"));
270 }
271 }
272 tw.addEndElement();
273 tw.endDocument();
274 result.write(tw.getResult());
275
276 logger.debug("Number of synthesized sound fragments: " + num);
277 logger.debug("audio encoding unreleased bytes : "
278 + mAudioFootprintMonitor.getUnreleasedEncondingMem());
279 logger.debug("TTS unreleased bytes: " + mAudioFootprintMonitor.getUnreleasedTTSMem());
280
281
282
283
284
285 tw = new TreeWriter(runtime);
286 tw.startDocument(runtime.getStaticBaseURI());
287 tw.addStartElement(StatusRootTag);
288 if (ssmltoaudio.getErrorRate() == 0)
289 tw.addAttribute(Status_attr_result, "ok");
290 else {
291 tw.addAttribute(Status_attr_result, "error");
292 tw.addAttribute(Status_attr_success_rate, (int)Math.floor(100 * (1 - ssmltoaudio.getErrorRate())) + "%");
293 }
294 tw.addEndElement();
295 tw.endDocument();
296 status.write(tw.getResult());
297
298
299
300
301 if (logEnabled) {
302 logger.info("Writing TTS log");
303 TreeWriter xmlLog = new TreeWriter(runtime);
304 xmlLog.startDocument(runtime.getStaticBaseURI());
305 xmlLog.addStartElement(LogRootTag);
306 xmlLog.startContent();
307 for (TTSLog.Error err : log.readonlyGeneralErrors()) {
308 writeXMLerror(xmlLog, err);
309 }
310 for (Map.Entry<String, TTSLog.Entry> entry : log.getEntries()) {
311 TTSLog.Entry le = entry.getValue();
312 xmlLog.addStartElement(LogTextTag);
313
314 xmlLog.addAttribute(Log_attr_id, entry.getKey());
315 if (le.getClip() != null) {
316 String basename = new File(le.getClip().src).getName();
317 xmlLog.addAttribute(Log_attr_file, basename);
318 xmlLog.addAttribute(Log_attr_begin, String.valueOf(le.getClip().clipBegin));
319 xmlLog.addAttribute(Log_attr_end, String.valueOf(le.getClip().clipEnd));
320 }
321
322 xmlLog.addAttribute(Log_attr_timeout, "" + le.getTimeout() + "s");
323
324 if (le.getSelectedVoice() != null)
325 xmlLog.addAttribute(Log_attr_selected_voice, le.getSelectedVoice()
326 .toString());
327 if (le.getActualVoice() != null) {
328 xmlLog.addAttribute(Log_attr_actual_voice, le.getActualVoice().toString());
329 xmlLog.addAttribute(Log_attr_time_elapsed, "" + le.getTimeElapsed() + "s");
330 }
331
332 for (TTSLog.Error err : le.getReadOnlyErrors())
333 writeXMLerror(xmlLog, err);
334
335 if (le.getSSML() != null) {
336 xmlLog.addStartElement(LogSsmlTag);
337 xmlLog.addSubtree(le.getSSML());
338 xmlLog.addEndElement();
339 }else{
340 xmlLog.addText("No SSML available. This piece of text has probably been extracted from inside another sentence.");
341 }
342
343 xmlLog.addEndElement();
344 }
345
346 xmlLog.addEndElement();
347 xmlLog.endDocument();
348
349 logOutput.write(xmlLog.getResult());
350 }
351 }
352
353 private static void writeXMLerror(TreeWriter tw, TTSLog.Error err) {
354 tw.addStartElement(LogErrorTag);
355 tw.addAttribute(Log_attr_code, err.getErrorCode().toString());
356 String message = err.getMessage();
357 if (err.getCause() != null)
358 message += ("\nError stack trace: " + getStack(err.getCause()));
359 tw.addText(message);
360 tw.addEndElement();
361 }
362
363 private static String getStack(Throwable t) {
364 StringWriter writer = new StringWriter();
365 PrintWriter printWriter = new PrintWriter(writer);
366 t.printStackTrace(printWriter);
367 printWriter.flush();
368 return writer.toString();
369 }
370 }