1 package org.daisy.pipeline.tts;
2
3 import java.io.PrintWriter;
4 import java.io.StringWriter;
5 import java.util.ArrayList;
6 import java.util.Collection;
7 import java.util.Collections;
8 import java.util.Comparator;
9 import java.util.HashMap;
10 import java.util.HashSet;
11 import java.util.LinkedHashMap;
12 import java.util.LinkedHashSet;
13 import java.util.List;
14 import java.util.Locale;
15 import java.util.Map;
16 import java.util.Map.Entry;
17 import java.util.Set;
18 import java.util.TreeSet;
19
20 import com.google.common.collect.Iterables;
21
22 import cz.vutbr.web.css.Term;
23
24 import org.daisy.pipeline.css.speech.VoiceFamilyList;
25 import org.daisy.pipeline.css.speech.VoiceFamilyList.VoiceFamily;
26 import org.daisy.pipeline.tts.TTSService.SynthesisException;
27 import org.daisy.pipeline.tts.VoiceInfo.Gender;
28 import org.daisy.pipeline.tts.VoiceInfo.LanguageRange;
29 import static org.daisy.pipeline.tts.VoiceInfo.NO_DEFINITE_GENDER;
30
31 import org.slf4j.Logger;
32 import org.slf4j.LoggerFactory;
33
34 public class VoiceManager {
35
36 private Logger ServerLogger = LoggerFactory.getLogger(VoiceManager.class);
37
38 private final Set<String> engineNames;
39
40
41
42 private final Map<Voice,TTSEngine> bestEngines;
43
44
45
46 private final Map<VoiceKey,Voice> primaryVoices;
47
48
49
50
51 private final Map<VoiceKey,Collection<Voice>> secondaryVoices;
52
53
54
55
56
57 private final Map<VoiceKey,Collection<LanguageRangeVoiceTuple>> voiceIndex;
58
59
60
61
62 private final Map<String,Voice> voiceForID;
63
64 public VoiceManager(Collection<TTSEngine> engines, Collection<VoiceInfo> voiceInfoFromConfig) {
65 engineNames = new HashSet<>();
66 for (TTSEngine e : engines)
67 engineNames.add(e.getProvider().getName().toLowerCase());
68
69
70 bestEngines = new LinkedHashMap<Voice,TTSEngine>();
71 voiceForID = new HashMap<>(); {
72 TTSTimeout timeout = new TTSTimeout();
73 int timeoutSecs = 30;
74
75
76 List<TTSEngine> sortedEngines = new ArrayList<>(engines);
77 Collections.sort(sortedEngines, Comparator.comparingInt(TTSEngine::getOverallPriority).reversed());
78 for (TTSEngine tts : sortedEngines) {
79 timeout.enableForCurrentThread(timeoutSecs);
80 try {
81 Collection<Voice> voices = tts.getAvailableVoices();
82 if (voices != null)
83 for (Voice v : voices) {
84 if (!bestEngines.containsKey(v))
85 bestEngines.put(v, tts);
86 if (!voiceForID.containsKey(v.getID()))
87 voiceForID.put(v.getID(), v);
88 }
89 } catch (SynthesisException e) {
90 ServerLogger.error("error while retrieving the voices of "
91 + tts.getProvider().getName());
92 ServerLogger.debug(tts.getProvider().getName()
93 + " getAvailableVoices error: " + getStack(e));
94 } catch (InterruptedException e) {
95 ServerLogger.error("timeout while retrieving the voices of "
96 + tts.getProvider().getName()
97 + " (exceeded " + timeoutSecs + " seconds)");
98 } finally {
99 timeout.disable();
100 }
101 }
102 timeout.close();
103 }
104
105
106 List<VoiceInfo> voiceInfo = new ArrayList<>();
107
108
109 primaryVoices = new HashMap<>();
110 for (Voice v : bestEngines.keySet()) {
111 primaryVoices.put(new VoiceKey(v.getEngine(), v.getName()), v);
112 if (!v.getLocale().isEmpty() && v.getGender().isPresent()) {
113 Gender g = v.getGender().get();
114 for (LanguageRange l : v.getLocale())
115 voiceInfo.add(new VoiceInfo(v.getEngine(), v.getName(), l, g, 0));
116 }
117 }
118
119
120 final float priorityVariantPenalty = 0.1f;
121 List<VoiceInfo> derivedVoiceInfo = new ArrayList<>(); {
122 for (VoiceInfo vi : voiceInfo) {
123 Locale shortLang = vi.language.getPrimaryLanguageSubTag();
124 if (shortLang != null && !shortLang.toString().equals(vi.language.toString()))
125 derivedVoiceInfo.add(
126 new VoiceInfo(vi.voiceEngine, vi.voiceName, shortLang, vi.gender,
127 vi.priority - priorityVariantPenalty));
128 }
129 }
130 voiceInfo.addAll(derivedVoiceInfo);
131
132
133
134
135
136 voiceInfo.addAll(voiceInfoFromConfig);
137
138
139
140 Collections.sort(voiceInfo, sortByPriority.thenComparing(sortBySpecificity));
141
142
143 List<VoiceInfo> availableVoiceInfo = new ArrayList<>(); {
144 for (VoiceInfo i : voiceInfo)
145 if (primaryVoices.containsKey(new VoiceKey(i.voiceEngine, i.voiceName)))
146 availableVoiceInfo.add(i);
147 }
148
149
150
151 secondaryVoices = new HashMap<>(); {
152 for (boolean sameEngine : new boolean[]{true, false})
153 for (boolean sameGender : new boolean[]{true, false})
154 for (VoiceInfo best : voiceInfo) {
155 VoiceKey bestKey = new VoiceKey(best.voiceEngine, best.voiceName);
156 for (VoiceInfo fallback : availableVoiceInfo) {
157 VoiceKey fallbackKey = new VoiceKey(fallback.voiceEngine, fallback.voiceName);
158 if (!fallbackKey.equals(bestKey))
159 if (best.language.getPrimaryLanguageSubTag() == null) {
160 if (!sameGender && !sameEngine) {
161
162
163
164
165 Collection<Voice> vv = secondaryVoices.get(bestKey);
166 if (vv == null) {
167 vv = new LinkedHashSet<>();
168 secondaryVoices.put(bestKey, vv);
169 }
170 vv.add(primaryVoices.get(fallbackKey));
171 }
172 } else if (fallback.language.equals(best.language)
173 && (!sameGender
174
175
176 || (fallback.gender != NO_DEFINITE_GENDER
177 && fallback.gender == best.gender))
178 && (!sameEngine
179 || fallback.voiceEngine.equals(best.voiceEngine))) {
180 Collection<Voice> vv = secondaryVoices.get(bestKey);
181 if (vv == null) {
182 vv = new LinkedHashSet<>();
183 secondaryVoices.put(bestKey, vv);
184 }
185 vv.add(primaryVoices.get(fallbackKey));
186 }
187 }
188 }
189 }
190
191
192 Set<Locale> allLangs = new HashSet<Locale>(); {
193 for (VoiceInfo vi : availableVoiceInfo) {
194 Locale shortLang = vi.language.getPrimaryLanguageSubTag();
195 if (shortLang != null)
196 allLangs.add(shortLang); }}
197 Set<Gender> allGenders = new HashSet<Gender>(); {
198 for (VoiceInfo vi : availableVoiceInfo) allGenders.add(vi.gender);
199 allGenders.remove(NO_DEFINITE_GENDER); }
200 voiceIndex = new HashMap<>(); {
201 for (VoiceInfo vi : availableVoiceInfo) {
202 if (vi.language.getPrimaryLanguageSubTag() == null)
203
204
205 for (Locale shortLang : allLangs)
206 for (boolean sameEngine : new boolean[]{true, false}) {
207 if (vi.gender == NO_DEFINITE_GENDER)
208
209
210 for (Gender g : allGenders) {
211 VoiceKey k = new VoiceKey(shortLang, g, sameEngine ? vi.voiceEngine : null);
212 Collection<LanguageRangeVoiceTuple> vv = voiceIndex.get(k);
213 if (vv == null) {
214 vv = new ArrayList<>();
215 voiceIndex.put(k, vv);
216 }
217 Voice v = primaryVoices.get(new VoiceKey(vi.voiceEngine, vi.voiceName));
218 vv.add(new LanguageRangeVoiceTuple(vi.language, v));
219 }
220 for (boolean sameGender : new boolean[]{true, false}) {
221 VoiceKey k = new VoiceKey(shortLang,
222 sameGender ? vi.gender : null,
223 sameEngine ? vi.voiceEngine : null);
224 Collection<LanguageRangeVoiceTuple> vv = voiceIndex.get(k);
225 if (vv == null) {
226 vv = new ArrayList<>();
227 voiceIndex.put(k, vv);
228 }
229 Voice v = primaryVoices.get(new VoiceKey(vi.voiceEngine, vi.voiceName));
230 vv.add(new LanguageRangeVoiceTuple(vi.language, v));
231 }
232 }
233 for (boolean sameLanguage : new boolean[]{true, false}) {
234 Locale shortLang = vi.language.getPrimaryLanguageSubTag();
235 if (shortLang == null) shortLang = VoiceKey.MUL;
236 for (boolean sameEngine : new boolean[]{true, false}) {
237 if (vi.gender == NO_DEFINITE_GENDER)
238 for (Gender g : allGenders) {
239 VoiceKey k = new VoiceKey(sameLanguage ? shortLang : null,
240 g,
241 sameEngine ? vi.voiceEngine : null);
242 Collection<LanguageRangeVoiceTuple> vv = voiceIndex.get(k);
243 if (vv == null) {
244 vv = new ArrayList<>();
245 voiceIndex.put(k, vv);
246 }
247 Voice v = primaryVoices.get(new VoiceKey(vi.voiceEngine, vi.voiceName));
248 vv.add(new LanguageRangeVoiceTuple(vi.language, v));
249 }
250 for (boolean sameGender : new boolean[]{true, false}) {
251 VoiceKey k = new VoiceKey(sameLanguage ? shortLang : null,
252 sameGender ? vi.gender : null,
253 sameEngine ? vi.voiceEngine : null);
254 Collection<LanguageRangeVoiceTuple> vv = voiceIndex.get(k);
255 if (vv == null) {
256 vv = new ArrayList<>();
257 voiceIndex.put(k, vv);
258 }
259 Voice v = primaryVoices.get(new VoiceKey(vi.voiceEngine, vi.voiceName));
260 vv.add(new LanguageRangeVoiceTuple(vi.language, v));
261 }
262 }
263 }
264 }
265 }
266
267
268 StringBuilder sb = new StringBuilder("Available voices:");
269 for (Entry<Voice,TTSEngine> e : bestEngines.entrySet())
270 sb.append("\n * ")
271 .append(e.getKey());
272 ServerLogger.debug(sb.toString());
273 sb = new StringBuilder("Voice selection data:");
274 Collection<VoiceInfo> sortedAvailableVoiceInfo = new TreeSet<VoiceInfo>(
275 new Comparator<VoiceInfo>() {
276 public int compare(VoiceInfo vi1, VoiceInfo vi2) {
277
278
279 if (vi1.language.getPrimaryLanguageSubTag() == null && vi2.language.getPrimaryLanguageSubTag() != null)
280 return 1;
281 else if (vi1.language.getPrimaryLanguageSubTag() != null && vi2.language.getPrimaryLanguageSubTag() == null)
282 return -1;
283 int compare = vi1.language.toString().compareTo(vi2.language.toString());
284 if (compare != 0) return compare;
285
286 compare = vi1.gender.compareTo(vi2.gender);
287 if (compare != 0) return compare;
288
289
290
291 if (vi1.voiceEngine.equals(vi2.voiceEngine) && vi1.voiceName.equals(vi2.voiceName))
292 return 0;
293
294 compare = Float.valueOf(vi2.priority).compareTo(Float.valueOf(vi1.priority));
295 if (compare != 0)
296 return compare;
297 compare = vi2.voiceEngine.compareTo(vi1.voiceEngine);
298 if (compare != 0)
299 return compare;
300 return vi2.voiceName.compareTo(vi1.voiceName);
301 }
302 }
303 );
304 sortedAvailableVoiceInfo.addAll(availableVoiceInfo);
305 for (VoiceInfo vi : sortedAvailableVoiceInfo)
306 sb.append("\n * {")
307 .append("locale:").append(vi.language)
308 .append(", gender:").append(vi.gender)
309 .append("}")
310 .append(" -> ")
311 .append(primaryVoices.get(new VoiceKey(vi.voiceEngine, vi.voiceName)));
312 ServerLogger.debug(sb.toString());
313 }
314
315
316
317
318
319
320 public TTSEngine getTTS(Voice voice) {
321 return bestEngines.get(voice);
322 }
323
324
325
326
327 public Voice getVoiceForID(String id) {
328 return voiceForID.get(id);
329 }
330
331
332
333
334
335
336
337
338
339 public Voice findAvailableVoice(String voiceEngine, String voiceName, Locale lang, Gender gender) {
340 return Iterables.getFirst(findAvailableVoices(voiceEngine, voiceName, lang, gender), null);
341 }
342
343 public Iterable<Voice> findAvailableVoices(Locale lang, VoiceFamilyList voiceFamily) {
344 if (voiceFamily == null)
345 return findAvailableVoices(null, null, lang, null);
346 Collection<Voice> voices = new LinkedHashSet<>();
347 Collection<Voice> nonExactMatches = new LinkedHashSet<>();
348 for (Term<?> term : voiceFamily) {
349 VoiceFamily f = (VoiceFamily)term;
350 String engineName = null;
351 String voiceName = null;
352 Gender gender = null;
353 if (f.getFamilyName().isPresent()) {
354 String name = f.getFamilyName().get();
355 if (engineNames.contains(name.toLowerCase()))
356 engineName = name;
357 else
358 voiceName = name;
359 }
360 if (f.getGender().isPresent())
361 gender = Gender.of(f.getGender().get(), f.getAge());
362 boolean exact = true;
363 Integer variant = f.getVariant().orElse(null);
364 int k = 0;
365 for (Voice v : findAvailableVoices(engineName, voiceName, lang, gender)) {
366 k++;
367 if ((exact = exact && matches(v, engineName, voiceName, lang, gender))
368 && (variant == null || k == variant))
369 voices.add(v);
370 else
371 nonExactMatches.add(v);
372 }
373 }
374 voices.addAll(nonExactMatches);
375 return voices;
376 }
377
378 private final Map<VoiceKey,Iterable<Voice>> cache = new HashMap<>();
379
380 public Iterable<Voice> findAvailableVoices(String voiceEngine, String voiceName, Locale lang, Gender gender) {
381 VoiceKey cacheKey = new VoiceKey(voiceEngine, voiceName, lang, gender);
382 Iterable<Voice> fromCache = cache.get(cacheKey);
383 if (fromCache != null)
384 return fromCache;
385 Collection<Voice> voices = new LinkedHashSet<>();
386 if (voiceEngine != null && !voiceEngine.isEmpty() && voiceName != null && !voiceName.isEmpty()) {
387 VoiceKey preferred = new VoiceKey(voiceEngine, voiceName);
388
389 Voice primary = primaryVoices.get(preferred);
390 if (primary != null)
391 voices.add(primary);
392 Collection<Voice> fallback = secondaryVoices.get(preferred);
393 if (fallback != null)
394 voices.addAll(fallback);
395 }
396 if (lang != null || gender != null || voiceName == null) {
397 Locale shortLang = lang != null ? new Locale(lang.getLanguage()) : lang;
398
399
400
401 addExactMatches(voices, lang, shortLang, gender, voiceEngine);
402 if (gender != null && gender != NO_DEFINITE_GENDER)
403 addExactMatches(voices, lang, shortLang, NO_DEFINITE_GENDER, voiceEngine);
404 if (lang != null) {
405 if (!lang.equals(shortLang)) {
406 addExactMatches(voices, shortLang, shortLang, gender, voiceEngine);
407 if (gender != null && gender != NO_DEFINITE_GENDER)
408 addExactMatches(voices, shortLang, shortLang, NO_DEFINITE_GENDER, voiceEngine);
409 }
410 if (!lang.equals(VoiceKey.MUL)) {
411 addExactMatches(voices, VoiceKey.MUL, VoiceKey.MUL, gender, voiceEngine);
412 if (gender != null && gender != NO_DEFINITE_GENDER)
413 addExactMatches(voices, VoiceKey.MUL, VoiceKey.MUL, NO_DEFINITE_GENDER, voiceEngine);
414 }
415 }
416 if (gender != null) {
417 addExactMatches(voices, lang, shortLang, null, voiceEngine);
418 if (lang != null) {
419 if (!lang.equals(shortLang))
420 addExactMatches(voices, shortLang, shortLang, null, voiceEngine);
421 if (!lang.equals(VoiceKey.MUL))
422 addExactMatches(voices, VoiceKey.MUL, VoiceKey.MUL, null, voiceEngine);
423 }
424 }
425 if (voiceEngine != null) {
426 addExactMatches(voices, lang, shortLang, gender, null);
427 if (gender != null && gender != NO_DEFINITE_GENDER)
428 addExactMatches(voices, lang, shortLang, NO_DEFINITE_GENDER, null);
429 if (lang != null) {
430 if (!lang.equals(shortLang)) {
431 addExactMatches(voices, shortLang, shortLang, gender, null);
432 if (gender != null && gender != NO_DEFINITE_GENDER)
433 addExactMatches(voices, shortLang, shortLang, NO_DEFINITE_GENDER, null);
434 }
435 if (!lang.equals(VoiceKey.MUL)) {
436 addExactMatches(voices, VoiceKey.MUL, VoiceKey.MUL, gender, null);
437 if (gender != null && gender != NO_DEFINITE_GENDER)
438 addExactMatches(voices, VoiceKey.MUL, VoiceKey.MUL, NO_DEFINITE_GENDER, null);
439 }
440 }
441 }
442 if (gender != null && voiceEngine != null) {
443 addExactMatches(voices, lang, shortLang, null, null);
444 if (lang != null) {
445 if (!lang.equals(shortLang))
446 addExactMatches(voices, shortLang, shortLang, null, null);
447 if (!lang.equals(VoiceKey.MUL))
448 addExactMatches(voices, VoiceKey.MUL, VoiceKey.MUL, null, null);
449 }
450 }
451 if (voiceName != null) {
452
453
454
455 voices = new ArrayList<>(voices);
456 Collections.sort((List<Voice>)voices,
457 Comparator.<Voice,Boolean>comparing(x -> !x.getName().equals(voiceName)));
458 }
459 }
460 cache.put(cacheKey, voices);
461 return voices;
462 }
463
464 private void addExactMatches(Collection<Voice> collect, Locale lang, Locale shortLang, Gender gender, String voiceEngine) {
465 Collection<LanguageRangeVoiceTuple> vv = voiceIndex.get(new VoiceKey(shortLang, gender, voiceEngine));
466 if (vv != null)
467 for (LanguageRangeVoiceTuple v : vv)
468 if (lang == null || v.language.matches(lang))
469 collect.add(v.voice);
470 }
471
472
473
474
475 public boolean matches(Voice voice, String voiceEngine, String voiceName, Locale lang, Gender gender) {
476 if (voiceEngine != null && !voiceEngine.equalsIgnoreCase(voice.getEngine()))
477 return false;
478 if (voiceName != null && !voiceName.equalsIgnoreCase(voice.getName()))
479 return false;
480 if (lang == null && gender == null)
481 return true;
482 for (VoiceKey k : voiceIndex.keySet()) {
483 if (k.lang != null && k.gender != null && k.engine != null)
484 if (gender == null || k.gender == NO_DEFINITE_GENDER || k.gender.equals(gender))
485 for (LanguageRangeVoiceTuple v : voiceIndex.get(k))
486 if (v.voice.equals(voice))
487 if (lang == null || v.language.matches(lang))
488 return true;
489 }
490 return false;
491 }
492
493 public boolean matches(Voice voice, Locale lang, VoiceFamilyList voiceFamily) {
494 if (voiceFamily == null)
495 return matches(voice, null, null, lang, null);
496 for (Term<?> term : voiceFamily)
497 if (matches(voice, lang, (VoiceFamily)term))
498 return true;
499 return false;
500 }
501
502 private boolean matches(Voice voice, Locale lang, VoiceFamily voiceFamily) {
503 if (voiceFamily == null)
504 return matches(voice, null, null, lang, null);
505 if (voiceFamily.getFamilyName().isPresent()) {
506 String name = voiceFamily.getFamilyName().get();
507 if (engineNames.contains(name.toLowerCase()))
508 return matches(voice, name, null, lang, null);
509 else
510 return matches(voice, null, name, lang, null);
511 } else
512 return matches(voice, null, null, lang, Gender.of(voiceFamily.getGender().get()));
513 }
514
515 private static String getStack(Throwable t) {
516 StringWriter writer = new StringWriter();
517 PrintWriter printWriter = new PrintWriter(writer);
518 t.printStackTrace(printWriter);
519 printWriter.flush();
520 return writer.toString();
521 }
522
523
524 private static final Comparator<VoiceInfo> sortByPriority = new Comparator<VoiceInfo>() {
525 public int compare(VoiceInfo vi1, VoiceInfo vi2) {
526 return Float.valueOf(vi2.priority).compareTo(Float.valueOf(vi1.priority));
527 }
528 };
529
530
531 private static final Comparator<VoiceInfo> sortBySpecificity
532 = Comparator.<VoiceInfo>comparingInt(vi -> vi.language.getSpecificity()).reversed();
533
534 private static class VoiceKey {
535
536 private static final Locale MUL = new Locale("mul");
537
538 private final String engine;
539 private final String name;
540
541
542
543
544
545 private final Locale lang;
546 private final Gender gender;
547
548
549
550
551 public VoiceKey(String engine, String name) {
552 this.engine = engine == null ? null : engine.toLowerCase();
553 this.name = name == null ? null : name.toLowerCase();
554 this.lang = null;
555 this.gender = null;
556 }
557
558
559
560
561
562
563 public VoiceKey(Locale primaryLang, Gender gender, String engine) {
564 this.lang = primaryLang;
565 this.gender = gender;
566 this.engine = engine == null ? null : engine.toLowerCase();
567 this.name = null;
568 }
569
570
571
572
573 public VoiceKey(String engine, String name, Locale lang, Gender gender) {
574 this.engine = engine;
575 this.name = name;
576 this.lang = lang;
577 this.gender = gender;
578 }
579
580 @Override
581 public int hashCode() {
582 int res = 0;
583 if (this.lang != null)
584 res ^= this.lang.hashCode();
585 if (this.gender != null)
586 res ^= this.gender.hashCode();
587 if (this.engine != null)
588 res ^= this.engine.hashCode();
589 if (this.name != null)
590 res ^= this.name.hashCode();
591 return res;
592 }
593
594 @Override
595 public boolean equals(Object other) {
596 VoiceKey o = (VoiceKey) other;
597 return (lang == o.lang || (lang != null && lang.equals(o.lang)))
598 && (gender == o.gender || (gender != null && gender.equals(o.gender)))
599 && (engine == o.engine || (engine != null && engine.equals(o.engine)))
600 && (name == o.name || (name != null && name.equals(o.name)));
601 }
602
603 @Override
604 public String toString() {
605 StringBuilder s = new StringBuilder();
606 s.append("{");
607 if (engine != null)
608 s.append("engine: " + engine);
609 if (name != null)
610 s.append(", name: " + name);
611 if (lang != null)
612 s.append(", lang: " + lang);
613 if (gender != null)
614 s.append(", gender: " + gender);
615 if (s.length() > 1 && s.charAt(1) == ',')
616 s.delete(1, 3);
617 s.append("}");
618 return s.toString();
619 }
620 }
621
622 private static class LanguageRangeVoiceTuple {
623
624 private final LanguageRange language;
625 private final Voice voice;
626
627 public LanguageRangeVoiceTuple(LanguageRange language, Voice voice) {
628 this.language = language;
629 this.voice = voice;
630 }
631
632 @Override
633 public String toString() {
634 StringBuilder s = new StringBuilder();
635 s.append(language).append(": ").append(voice);
636 return s.toString();
637 }
638 }
639 }