1 package org.daisy.pipeline.audio.calabash.impl;
2
3 import java.io.File;
4 import java.io.IOException;
5 import java.math.BigDecimal;
6 import java.math.MathContext;
7 import java.nio.file.Files;
8 import java.net.URI;
9 import java.net.URISyntaxException;
10 import java.time.Duration;
11 import java.util.ArrayList;
12 import java.util.Comparator;
13 import java.util.HashMap;
14 import java.util.Iterator;
15 import java.util.List;
16 import java.util.Map;
17 import java.util.NoSuchElementException;
18 import java.util.Optional;
19 import java.util.SortedMap;
20 import java.util.TreeMap;
21
22 import javax.sound.sampled.AudioFileFormat;
23 import javax.sound.sampled.AudioInputStream;
24 import javax.sound.sampled.UnsupportedAudioFileException;
25 import javax.xml.namespace.QName;
26 import static javax.xml.stream.XMLStreamConstants.END_DOCUMENT;
27 import static javax.xml.stream.XMLStreamConstants.END_ELEMENT;
28 import static javax.xml.stream.XMLStreamConstants.START_DOCUMENT;
29 import static javax.xml.stream.XMLStreamConstants.START_ELEMENT;
30 import javax.xml.stream.XMLStreamException;
31
32 import com.google.common.collect.ImmutableMap;
33 import static com.google.common.io.Files.getFileExtension;
34
35 import com.xmlcalabash.core.XProcRuntime;
36 import com.xmlcalabash.io.ReadablePipe;
37 import com.xmlcalabash.io.WritablePipe;
38 import com.xmlcalabash.library.DefaultStep;
39 import com.xmlcalabash.model.RuntimeValue;
40 import com.xmlcalabash.runtime.XAtomicStep;
41
42 import net.sf.saxon.s9api.SaxonApiException;
43
44 import org.daisy.common.messaging.MessageAppender;
45 import org.daisy.common.messaging.MessageBuilder;
46 import org.daisy.common.stax.BaseURIAwareXMLStreamReader;
47 import org.daisy.common.transform.InputValue;
48 import org.daisy.common.transform.OutputValue;
49 import org.daisy.common.transform.TransformerException;
50 import org.daisy.common.transform.XMLInputValue;
51 import org.daisy.common.transform.XMLOutputValue;
52 import org.daisy.common.transform.XMLTransformer;
53 import org.daisy.common.xproc.calabash.XMLCalabashInputValue;
54 import org.daisy.common.xproc.calabash.XMLCalabashOutputValue;
55 import org.daisy.common.xproc.calabash.XProcStep;
56 import org.daisy.common.xproc.calabash.XProcStepProvider;
57 import org.daisy.common.xproc.XProcMonitor;
58 import static org.daisy.pipeline.file.FileUtils.normalizeURI;
59 import org.daisy.pipeline.fileset.Fileset;
60 import org.daisy.pipeline.audio.AudioClip;
61 import org.daisy.pipeline.audio.AudioDecoder;
62 import org.daisy.pipeline.audio.AudioEncoder;
63 import org.daisy.pipeline.audio.AudioFileTypes;
64 import org.daisy.pipeline.audio.AudioServices;
65 import org.daisy.pipeline.audio.AudioUtils;
66 import org.daisy.pipeline.audio.PCMAudioFormat;
67
68 import org.osgi.service.component.annotations.Component;
69 import org.osgi.service.component.annotations.Reference;
70 import org.osgi.service.component.annotations.ReferenceCardinality;
71 import org.osgi.service.component.annotations.ReferencePolicy;
72
73 public class AudioRearrangeStep extends DefaultStep implements XProcStep {
74
75 @Component(
76 name = "px:audio-rearrange",
77 service = { XProcStepProvider.class },
78 property = { "type:String={http://www.daisy.org/ns/pipeline/xproc}audio-rearrange" }
79 )
80 public static class Provider implements XProcStepProvider {
81
82 private AudioServices audioServices;
83
84 @Override
85 public XProcStep newStep(XProcRuntime runtime, XAtomicStep step, XProcMonitor monitor, Map<String,String> properties) {
86 return new AudioRearrangeStep(runtime, step, audioServices);
87 }
88
89 @Reference(
90 name = "AudioServices",
91 unbind = "-",
92 service = AudioServices.class,
93 cardinality = ReferenceCardinality.MANDATORY,
94 policy = ReferencePolicy.STATIC
95 )
96 protected void setAudioServices(AudioServices audioServices) {
97 this.audioServices = audioServices;
98 }
99 }
100
101 private final static QName _SOURCE = new QName("source");
102 private final static QName _RESULT = new QName("result");
103 private final static QName _DESIRED = new QName("desired");
104 private final static QName _TEMP_FILES = new QName("temp-files");
105
106 private ReadablePipe sourcePipe = null;
107 private ReadablePipe desiredPipe = null;
108 private WritablePipe resultPipe = null;
109 private WritablePipe tempFilesPipe = null;
110
111 private final AudioServices audioServices;
112 private String tempDirOption = null;
113
114 private AudioRearrangeStep(XProcRuntime runtime, XAtomicStep step, AudioServices audioServices) {
115 super(runtime, step);
116 this.audioServices = audioServices;
117 }
118
119 @Override
120 public void setInput(String port, ReadablePipe pipe) {
121 if (_SOURCE.getLocalPart().equals(port))
122 sourcePipe = pipe;
123 else
124 desiredPipe = pipe;
125 }
126
127 @Override
128 public void setOutput(String port, WritablePipe pipe) {
129 if (_RESULT.getLocalPart().equals(port))
130 resultPipe = pipe;
131 else if (_TEMP_FILES.getLocalPart().equals(port))
132 tempFilesPipe = pipe;
133 }
134
135 @Override
136 public void setOption(net.sf.saxon.s9api.QName name, RuntimeValue value) {
137 if ("temp-dir".equals(name.getLocalName()))
138 tempDirOption = value.getString();
139 }
140
141 @Override
142 public void reset() {
143 sourcePipe.resetReader();
144 desiredPipe.resetReader();
145 resultPipe.resetWriter();
146 tempFilesPipe.resetWriter();
147 }
148
149 @Override
150 public void run() throws SaxonApiException {
151 super.run();
152 try {
153 new AudioRearrange()
154 .transform(
155 ImmutableMap.of(_SOURCE, new XMLCalabashInputValue(sourcePipe),
156 _DESIRED, new XMLCalabashInputValue(desiredPipe)),
157 ImmutableMap.of(_RESULT, new XMLCalabashOutputValue(resultPipe, runtime),
158 _TEMP_FILES, new XMLCalabashOutputValue(tempFilesPipe, runtime)))
159 .run();
160 } catch (Throwable e) {
161 throw XProcStep.raiseError(e, step);
162 }
163 }
164
165 private static final QName D_CLIP = new QName("http://www.daisy.org/ns/pipeline/data", "clip", "d");
166 private static final QName _CLIPBEGIN = new QName("clipBegin");
167 private static final QName _CLIPEND = new QName("clipEnd");
168 private static final QName _ORIGINAL_CLIPBEGIN = new QName("original-clipBegin");
169 private static final QName _ORIGINAL_CLIPEND = new QName("original-clipEnd");
170
171 private class AudioRearrange implements XMLTransformer {
172
173 public Runnable transform(Map<QName,InputValue<?>> input, Map<QName,OutputValue<?>> output) {
174 input = XMLTransformer.validateInput(input, ImmutableMap.of( _SOURCE, InputType.MANDATORY_NODE_SINGLE,
175 _DESIRED, InputType.MANDATORY_NODE_SINGLE));
176 output = XMLTransformer.validateOutput(output, ImmutableMap.of(_RESULT, OutputType.NODE_SEQUENCE,
177 _TEMP_FILES, OutputType.NODE_SEQUENCE));
178 XMLInputValue<?> sourceXML = (XMLInputValue<?>)input.get(_SOURCE);
179 XMLInputValue<?> desiredXML = (XMLInputValue<?>)input.get(_DESIRED);
180 XMLOutputValue<?> resultXML = (XMLOutputValue<?>)output.get(_RESULT);
181 XMLOutputValue<?> tempFilesXML = (XMLOutputValue<?>)output.get(_TEMP_FILES);
182 File tempDir; {
183 if (tempDirOption != null && !tempDirOption.isEmpty()) {
184 try {
185 tempDir = new File(new URI(tempDirOption));
186 } catch (URISyntaxException e) {
187 throw new RuntimeException("temp-dir option invalid: " + tempDirOption);
188 }
189 if (tempDir.exists() && tempDir.listFiles().length > 0)
190 throw new RuntimeException("temp-dir option must be a non-existing directory: " + tempDirOption);
191 } else {
192 try {
193 tempDir = Files.createTempDirectory("pipeline-").toFile();
194 tempDir.deleteOnExit();
195 } catch (IOException e) {
196 throw new TransformerException(e);
197 }
198 }
199 }
200 return () -> {
201 Map<URI,Fileset.File> source;
202 SortedMap<AudioClip,AudioClip> desired;
203 String resultMediaType;
204 URI resultBaseURI;
205 URI resultXmlBase; {
206 try {
207 source = new HashMap<>();
208 for (Fileset.File f : Fileset.unmarshall(sourceXML.asXMLStreamReader()))
209 if (!source.containsKey(f.href))
210 source.put(f.href, f);
211 desired = new TreeMap<>(new Comparator<AudioClip>() {
212 public int compare(AudioClip c1, AudioClip c2) {
213 int r = c1.src.compareTo(c2.src);
214 if (r != 0) return r;
215 r = c1.clipBegin.compareTo(c2.clipBegin);
216 if (r != 0) return r;
217 return c1.clipEnd.compareTo(c2.clipEnd); }});
218 resultMediaType = null;
219 BaseURIAwareXMLStreamReader xml = desiredXML.asXMLStreamReader();
220 URI filesetBase = xml.getBaseURI();
221 boolean hasXmlBase = false;
222 int depth = 0;
223 URI href = null;
224 URI originalHref = null;
225 document: while (true) {
226 try {
227 int event = xml.next();
228 switch (event) {
229 case START_DOCUMENT:
230 break;
231 case END_DOCUMENT:
232 break document;
233 case START_ELEMENT:
234 if (depth == 0 && Fileset.XMLConstants.D_FILESET.equals(xml.getName())) {
235 for (int i = 0; i < xml.getAttributeCount(); i++)
236 if (Fileset.XMLConstants.XML_BASE.equals(xml.getAttributeName(i))) {
237 filesetBase = filesetBase.resolve(xml.getAttributeValue(i));
238 hasXmlBase = true;
239 break;
240 }
241 depth++;
242 break;
243 } else if (depth == 1 && Fileset.XMLConstants.D_FILE.equals(xml.getName())) {
244 href = null;
245 originalHref = null;
246 String mediaType = null;
247 String originalMediaType = null;
248 for (int i = 0; i < xml.getAttributeCount(); i++)
249 if (Fileset.XMLConstants._HREF.equals(xml.getAttributeName(i)))
250 href = filesetBase.resolve(xml.getAttributeValue(i));
251 else if (Fileset.XMLConstants._ORIGINAL_HREF.equals(xml.getAttributeName(i)))
252 originalHref = filesetBase.resolve(xml.getAttributeValue(i));
253 else if (Fileset.XMLConstants._MEDIA_TYPE.equals(xml.getAttributeName(i)))
254 mediaType = xml.getAttributeValue(i);
255 else if (Fileset.XMLConstants._MEDIA_TYPE.equals(xml.getAttributeName(i)))
256 originalMediaType = xml.getAttributeValue(i);
257 if (mediaType != null && href != null && originalHref != null) {
258 originalHref = normalizeURI(originalHref);
259 if (source.containsKey(originalHref)) {
260 if (resultMediaType == null) {
261 resultMediaType = mediaType;
262 if (AudioFileTypes.fromMediaType(mediaType) == null)
263 throw new TransformerException(
264 new RuntimeException("media-type is not a recognized mime type: " + mediaType));
265 } else if (!resultMediaType.equals(mediaType))
266 throw new TransformerException(
267 new RuntimeException("media-type of all files must be the same, but got '"
268 + resultMediaType + "' and '" + mediaType + "'"));
269 if (originalMediaType != null
270 && AudioFileTypes.fromMediaType(originalMediaType) == null)
271 throw new TransformerException(
272 new RuntimeException("media-type is not a recognized mime type: " + originalMediaType));
273 href = normalizeURI(href);
274 try {
275 new File(href);
276 depth++;
277 break;
278 } catch (IllegalArgumentException e) {
279
280 }
281 }
282 }
283 } else if (depth == 2 && D_CLIP.equals(xml.getName())) {
284 BigDecimal clipBegin = null;
285 BigDecimal clipEnd = null;
286 BigDecimal originalClipBegin = null;
287 BigDecimal originalClipEnd = null;
288 for (int i = 0; i < xml.getAttributeCount(); i++)
289 try {
290 if (_CLIPBEGIN.equals(xml.getAttributeName(i)))
291 clipBegin = new BigDecimal(xml.getAttributeValue(i));
292 else if (_CLIPEND.equals(xml.getAttributeName(i)))
293 clipEnd = new BigDecimal(xml.getAttributeValue(i));
294 else if (_ORIGINAL_CLIPBEGIN.equals(xml.getAttributeName(i)))
295 originalClipBegin = new BigDecimal(xml.getAttributeValue(i));
296 else if (_ORIGINAL_CLIPEND.equals(xml.getAttributeName(i)))
297 originalClipEnd = new BigDecimal(xml.getAttributeValue(i));
298 } catch (NumberFormatException e) {
299 }
300 if (clipBegin != null && clipEnd != null && originalClipBegin != null && originalClipEnd != null)
301 desired.put(
302 new AudioClip(
303 href,
304 Duration.ofMillis(clipBegin.movePointRight(3).longValue()),
305 Duration.ofMillis(clipEnd.movePointRight(3).longValue())),
306 new AudioClip(
307 originalHref,
308 Duration.ofMillis(originalClipBegin.movePointRight(3).longValue()),
309 Duration.ofMillis(originalClipEnd.movePointRight(3).longValue())));
310 }
311 {
312 int d = depth + 1;
313 element: while (true) {
314 event = xml.next();
315 switch (event) {
316 case START_ELEMENT:
317 d++;
318 break;
319 case END_ELEMENT:
320 d--;
321 if (d == depth) break element;
322 default:
323 }
324 }
325 }
326 break;
327 case END_ELEMENT:
328 depth--;
329 break;
330 default:
331 }
332 } catch (NoSuchElementException e) {
333 break;
334 }
335 }
336 resultBaseURI = filesetBase;
337 resultXmlBase = hasXmlBase ? resultBaseURI : null;
338 } catch (XMLStreamException e) {
339 throw new TransformerException(e);
340 }
341
342 {
343 AudioClip prevClip = null;
344 for (AudioClip clip : desired.keySet()) {
345 if (prevClip == null) {
346 prevClip = clip;
347 continue;
348 }
349 if (prevClip.src.equals(clip.src) && clip.clipBegin.compareTo(prevClip.clipEnd) < 0)
350 throw new IllegalArgumentException("Invalid input: overlapping clips: " + prevClip + ", " + clip);
351 prevClip = clip;
352 }
353 }
354 }
355 List<Fileset.File> tempFiles = new ArrayList<>();
356 List<Fileset.File> result = new ArrayList<>(); {
357 if (!desired.isEmpty()) {
358 AudioFileFormat.Type resultFileType = AudioFileTypes.fromMediaType(resultMediaType);
359 Optional<AudioEncoder> encoder = audioServices.newEncoder(resultFileType, new HashMap<String,String>());
360 if (!encoder.isPresent())
361 throw new RuntimeException("No encoder found for file type " + resultFileType);
362
363
364 {
365 SortedMap<AudioClip,AudioClip> map = new TreeMap<>(desired.comparator());
366 AudioClip sourceClip = null;
367 AudioClip destinationClip = null;
368 for (Map.Entry<AudioClip,AudioClip> entry : desired.entrySet()) {
369 AudioClip s = entry.getValue();
370 AudioClip d = entry.getKey();
371 if (sourceClip != null
372 && s.src.equals(sourceClip.src) && s.clipBegin.equals(sourceClip.clipEnd)
373 && d.src.equals(destinationClip.src) && d.clipBegin.equals(destinationClip.clipEnd)) {
374 sourceClip = new AudioClip(sourceClip.src, sourceClip.clipBegin, s.clipEnd);
375 destinationClip = new AudioClip(destinationClip.src, destinationClip.clipBegin, d.clipEnd);
376 } else {
377 if (sourceClip != null)
378 map.put(destinationClip, sourceClip);
379 sourceClip = s;
380 destinationClip = d;
381 }
382 }
383 if (sourceClip != null)
384 map.put(destinationClip, sourceClip);
385 desired = map;
386 }
387 AudioClip prevSourceClip = null;
388 Fileset.File currentSourceFile = null;
389 PCMAudioFormat audioFormat = null;
390 URI audioFormatFromFile = null;
391 AudioInputStream currentSourcePCM = null;
392 long currentSourceElapsed = 0;
393 URI currentDestinationFile = null;
394 List<AudioInputStream> currentDestinationPCM = new ArrayList<>();
395 long currentDestinationElapsed = 0;
396 int fileCounter = 0;
397 MessageAppender progress = MessageAppender.getActiveBlock();
398 int progressUnit = 1;
399 int clipCounter = 0;
400 BigDecimal progressInc = new BigDecimal(progressUnit).divide(new BigDecimal(desired.size()), MathContext.DECIMAL128)
401 .min(BigDecimal.ONE);
402 for (Map.Entry<AudioClip,AudioClip> entry : desired.entrySet()) {
403 AudioClip destinationClip = entry.getKey();
404 AudioClip sourceClip = entry.getValue();
405 if (currentDestinationFile == null)
406 currentDestinationFile = destinationClip.src;
407 else if (!currentDestinationFile.equals(destinationClip.src)) {
408 File resultFile = new File(currentDestinationFile);
409 File tempFile = new File(tempDir, "tmp" + (++fileCounter) + "." + getFileExtension(resultFile.getName()));
410 if (tempDirOption == null || tempDirOption.isEmpty())
411 tempFile.deleteOnExit();
412 try {
413 tempFile.getParentFile().mkdirs();
414 encoder.get().encode(AudioUtils.concat(currentDestinationPCM), resultFileType, tempFile);
415 } catch (Throwable e) {
416 throw new TransformerException(
417 new IOException("Audio could not be written to file type: " + resultFileType, e));
418 }
419 Fileset.File f = Fileset.File.load(tempFile, Optional.of(resultMediaType));
420 tempFiles.add(f);
421 result.add(f.copy(resultFile.toURI()));
422 currentDestinationFile = destinationClip.src;
423 currentDestinationPCM = new ArrayList<>();
424 currentDestinationElapsed = 0;
425 }
426 if (currentSourceFile == null
427 || !currentSourceFile.href.equals(sourceClip.src)
428 || sourceClip.clipBegin.compareTo(prevSourceClip.clipEnd) < 0
429 ) {
430 currentSourceFile = source.get(sourceClip.src);
431 Optional<AudioFileFormat.Type> sourceFileType; {
432 Optional<String> sourceMediaType = currentSourceFile.mediaType;
433 if (sourceMediaType.isPresent())
434 sourceFileType = Optional.of(AudioFileTypes.fromMediaType(sourceMediaType.get()));
435 else
436 sourceFileType = Optional.empty();
437 }
438 Optional<AudioDecoder> decoder; {
439 if (sourceFileType.isPresent()) {
440 decoder = audioServices.newDecoder(sourceFileType.get(), new HashMap<String,String>());
441 if (!decoder.isPresent())
442 throw new TransformerException(
443 new RuntimeException("No decoder found for file type " + sourceFileType));
444 } else
445 decoder = audioServices.newDecoder(new HashMap<String,String>());
446 }
447 try {
448 currentSourcePCM = decoder.get().decode(currentSourceFile.read());
449 } catch (UnsupportedAudioFileException e) {
450 throw new TransformerException(
451 new IOException("Audio file could not be read (unsupported file type): "
452 + currentSourceFile.href, e));
453 } catch (Throwable e) {
454 throw new TransformerException(
455 new IOException("Audio file could not be read: " + currentSourceFile.href, e));
456 }
457 currentSourceElapsed = 0;
458 if (audioFormat == null) {
459 audioFormat = PCMAudioFormat.of(currentSourcePCM.getFormat());
460 audioFormatFromFile = currentSourceFile.href;
461 }
462 else if (!audioFormat.matches(currentSourcePCM.getFormat()))
463 throw new TransformerException(
464 new RuntimeException(
465 String.format("All input audio must have the same format, but got %s (%s) and %s (%s)",
466 audioFormat, audioFormatFromFile,
467 currentSourcePCM.getFormat(),
468 currentSourceFile.href)));
469 }
470
471 long sourceClipBegin = AudioUtils.getLengthInFrames(audioFormat, sourceClip.clipBegin);
472 long sourceClipEnd = AudioUtils.getLengthInFrames(audioFormat, sourceClip.clipEnd);
473 long destinationClipBegin = AudioUtils.getLengthInFrames(audioFormat, destinationClip.clipBegin);
474 long destinationClipEnd = AudioUtils.getLengthInFrames(audioFormat, destinationClip.clipEnd);
475 if (sourceClipBegin < currentSourceElapsed)
476 throw new IllegalStateException();
477
478 long currentSourceRemaining = currentSourcePCM.getFrameLength();
479 if (currentSourceRemaining == 0) {
480
481 } else {
482 AudioInputStream clipPCM; {
483 long sourceClipLength = sourceClipEnd - sourceClipBegin;
484 long destinationClipLength = destinationClipEnd - destinationClipBegin;
485 long clipLength = Math.min(sourceClipLength, destinationClipLength);
486 if (clipLength > currentSourceRemaining) {
487
488 clipLength = currentSourceRemaining;
489 }
490 Iterator<AudioInputStream> chunks = AudioUtils.split(
491 currentSourcePCM,
492 sourceClipBegin - currentSourceElapsed,
493 sourceClipBegin - currentSourceElapsed + clipLength
494 ).iterator();
495 currentSourceElapsed += chunks.next().getFrameLength();
496 clipPCM = chunks.next();
497 currentSourcePCM = chunks.next();
498 currentSourceElapsed += clipPCM.getFrameLength();
499 }
500 if (destinationClipBegin < currentDestinationElapsed)
501 throw new IllegalStateException();
502 AudioInputStream silence = AudioUtils.createSilence(
503 clipPCM.getFormat(),
504 destinationClipBegin - currentDestinationElapsed);
505 if (silence != null) {
506 currentDestinationPCM.add(silence);
507 currentDestinationElapsed += silence.getFrameLength();
508 }
509 currentDestinationPCM.add(clipPCM);
510 currentDestinationElapsed += clipPCM.getFrameLength();
511 }
512 prevSourceClip = sourceClip;
513 if (++clipCounter == progressUnit) {
514 clipCounter = 0;
515 progress.append(new MessageBuilder().withProgress(progressInc)).close();
516 }
517 }
518 progress.append(new MessageBuilder().withProgress(progressInc)).close();
519 File resultFile = new File(currentDestinationFile);
520 File tempFile = new File(tempDir, "tmp" + (++fileCounter) + "." + getFileExtension(resultFile.getName()));
521 if (tempDirOption == null || tempDirOption.isEmpty())
522 tempFile.deleteOnExit();
523 try {
524 tempFile.getParentFile().mkdirs();
525 encoder.get().encode(AudioUtils.concat(currentDestinationPCM), resultFileType, tempFile);
526 } catch (Throwable e) {
527 throw new TransformerException(
528 new IOException("Audio could not be written to file (type " + resultFileType + "): " + tempFile, e));
529 }
530 Fileset.File f = Fileset.File.load(tempFile, Optional.of(resultMediaType));
531 tempFiles.add(f);
532 result.add(f.copy(resultFile.toURI()));
533 }
534 }
535 resultXML.writeXMLStream(xml -> {
536 try {
537 xml.setBaseURI(resultBaseURI);
538 Fileset.marshall(xml, resultXmlBase, result);
539 } catch (XMLStreamException e) {
540 throw new TransformerException(e);
541 }
542 });
543 tempFilesXML.writeXMLStream(xml -> {
544 try {
545 URI base = tempDir.toURI();
546 xml.setBaseURI(base);
547 Fileset.marshall(xml, base, tempFiles);
548 } catch (XMLStreamException e) {
549 throw new TransformerException(e);
550 }
551 });
552
553 System.gc();
554 };
555 }
556 }
557 }