1 package org.daisy.pipeline.nlp.calabash.impl;
2
3 import java.util.ArrayList;
4 import java.util.Collection;
5 import java.util.Collections;
6 import java.util.function.Predicate;
7 import java.util.HashMap;
8 import java.util.HashSet;
9 import java.util.List;
10 import java.util.Locale;
11 import java.util.Map;
12 import java.util.Set;
13
14 import net.sf.saxon.s9api.QName;
15 import net.sf.saxon.s9api.SaxonApiException;
16 import net.sf.saxon.s9api.XdmNode;
17 import net.sf.saxon.s9api.XdmNodeKind;
18 import net.sf.saxon.sxpath.XPathExpression;
19 import net.sf.saxon.trans.XPathException;
20
21 import org.daisy.common.saxon.SaxonHelper;
22 import org.daisy.common.xproc.calabash.XProcStep;
23 import org.daisy.pipeline.nlp.DummyLangDetector;
24 import org.daisy.pipeline.nlp.LangDetector;
25 import org.daisy.pipeline.nlp.lexing.LexService.LexerInitException;
26 import org.daisy.pipeline.nlp.lexing.LexService.LexerToken;
27 import org.daisy.pipeline.nlp.lexing.LexServiceRegistry;
28
29 import com.xmlcalabash.core.XProcException;
30 import com.xmlcalabash.core.XProcRuntime;
31 import com.xmlcalabash.io.ReadablePipe;
32 import com.xmlcalabash.io.WritablePipe;
33 import com.xmlcalabash.library.DefaultStep;
34 import com.xmlcalabash.model.RuntimeValue;
35 import com.xmlcalabash.runtime.XAtomicStep;
36 import com.xmlcalabash.util.TreeWriter;
37
38 import org.slf4j.Logger;
39 import org.slf4j.LoggerFactory;
40
41
42
43
44
45 public class BreakDetectStep extends DefaultStep implements TreeWriterFactory, InlineSectionProcessor, XProcStep {
46
47 private Logger mLogger = LoggerFactory.getLogger(BreakDetectStep.class);
48 private ReadablePipe mSource = null;
49 private WritablePipe mResult = null;
50 private XProcRuntime mRuntime = null;
51 private LexServiceRegistry mLexerRegistry;
52 private Set<Locale> mLangs;
53 private RuntimeValue inlineTagsOption;
54 private RuntimeValue wordBeforeOption;
55 private RuntimeValue wordAfterOption;
56 private RuntimeValue sentenceBeforeOption;
57 private RuntimeValue sentenceAfterOption;
58 private QName wordTagOption;
59 private QName sentenceTagOption;
60 private LangDetector mLangDetector = null;
61
62 public BreakDetectStep(XProcRuntime runtime, XAtomicStep step, LexServiceRegistry registry) {
63 super(runtime, step);
64 mRuntime = runtime;
65 mLexerRegistry = registry;
66 }
67
68 public void setInput(String port, ReadablePipe pipe) {
69 if ("source".equals(port)) {
70 mSource = pipe;
71 }
72 }
73
74 @Override
75 public void setOption(QName name, RuntimeValue value) {
76 super.setOption(name, value);
77 if ("inline-tags".equalsIgnoreCase(name.getLocalName())) {
78 inlineTagsOption = value;
79 } else if ("ensure-word-before".equalsIgnoreCase(name.getLocalName())) {
80 wordBeforeOption = value;
81 } else if ("ensure-word-after".equalsIgnoreCase(name.getLocalName())) {
82 wordAfterOption = value;
83 } else if ("ensure-sentence-before".equalsIgnoreCase(name.getLocalName())) {
84 sentenceBeforeOption = value;
85 } else if ("ensure-sentence-after".equalsIgnoreCase(name.getLocalName())) {
86 sentenceAfterOption = value;
87 } else if ("output-word-tag".equalsIgnoreCase(name.getLocalName())) {
88 wordTagOption = value.getQName();
89 } else if ("output-sentence-tag".equalsIgnoreCase(name.getLocalName())) {
90 sentenceTagOption = value.getQName();
91 } else {
92 runtime.error(new RuntimeException("unrecognized option " + name));
93 }
94 }
95
96 public void setOutput(String port, WritablePipe pipe) {
97 mResult = pipe;
98 }
99
100 public void reset() {
101 mSource.resetReader();
102 mResult.resetWriter();
103 }
104
105 private Predicate<XdmNode> processXSLTMatchPatternOption(RuntimeValue option) throws XPathException {
106 if (!option.getString().equals("")) {
107 XPathExpression matcher = SaxonHelper.compileExpression(
108 option.getString(),
109 option.getNamespaceBindings(),
110 runtime.getProcessor().getUnderlyingConfiguration());
111 return n -> n.getNodeKind() == XdmNodeKind.ELEMENT && SaxonHelper.evaluateBoolean(matcher, n);
112 } else
113 return n -> false;
114 }
115
116 public void run() throws SaxonApiException {
117 super.run();
118
119 HashMap<Locale, LexerToken> langToToken = new HashMap<Locale, LexerToken>();
120
121
122 LexerToken generic;
123 try {
124 generic = mLexerRegistry.getFallbackToken(Collections.EMPTY_LIST);
125 } catch (LexerInitException e1) {
126 throw XProcStep.raiseError(e1, step);
127 }
128 langToToken.put(null, generic);
129
130 FormatSpecifications formatSpecs;
131 try {
132 formatSpecs = new FormatSpecifications(
133 sentenceTagOption, wordTagOption, "http://www.w3.org/XML/1998/namespace", "lang",
134 processXSLTMatchPatternOption(inlineTagsOption),
135 processXSLTMatchPatternOption(wordBeforeOption),
136 processXSLTMatchPatternOption(wordAfterOption),
137 processXSLTMatchPatternOption(sentenceBeforeOption),
138 processXSLTMatchPatternOption(sentenceAfterOption));
139 } catch (XPathException e) {
140 throw XProcStep.raiseError(e, step);
141 }
142
143 XmlBreakRebuilder xmlRebuilder = new XmlBreakRebuilder();
144
145 long before = System.currentTimeMillis();
146
147 if (mLangDetector == null) {
148 mLangDetector = new DummyLangDetector();
149 mLangDetector.train();
150 }
151
152 while (mSource.moreDocuments()) {
153 XdmNode doc = mSource.read();
154
155
156 mLangs = new HashSet<Locale>();
157 try {
158 new InlineSectionFinder().find(doc, 0, formatSpecs, this,
159 Collections.EMPTY_SET);
160 for (Locale lang : mLangs) {
161 if (!langToToken.containsKey(lang)) {
162 LexerToken token = mLexerRegistry.getTokenForLang(lang, langToToken
163 .values());
164 if (token == null) {
165 throw new LexerInitException(
166 "cannot find a lexer for the language: " + lang);
167 }
168 langToToken.put(lang, token);
169 }
170 }
171 } catch (LexerInitException e) {
172 throw XProcStep.raiseError(e, step);
173 }
174
175 mLogger.debug("Total number of language(s): " + (langToToken.size() - 1));
176 for (Map.Entry<Locale, LexerToken> entry : langToToken.entrySet()) {
177 mLogger.debug("LexService for language '"
178 + (entry.getKey() == null ? "<ANY>" : entry.getKey()) + "': "
179 + entry.getValue().getLexService().getName());
180 }
181
182
183 List<String> parsingErrors = new ArrayList<String>();
184 XdmNode tree;
185 try {
186 tree = xmlRebuilder.rebuild(this, langToToken, doc, formatSpecs,
187 mLangDetector, false, parsingErrors);
188 mResult.write(tree);
189 } catch (LexerInitException e) {
190 throw XProcStep.raiseError(e, step);
191 }
192 for (String error : parsingErrors) {
193 mRuntime.info(null, null, doc.getBaseURI() + ": " + error);
194 }
195 }
196
197 for (LexerToken token : langToToken.values()) {
198 mLexerRegistry.releaseToken(token);
199 }
200
201 long after = System.currentTimeMillis();
202 mLogger.debug("lexing time = " + (after - before) / 1000.0 + " s.");
203
204 mLangs = null;
205 }
206
207 @Override
208 public TreeWriter newInstance() {
209 return new TreeWriter(mRuntime);
210 }
211
212 @Override
213 public void onInlineSectionFound(List<Leaf> leaves, List<String> text, Locale lang)
214 throws LexerInitException {
215
216
217
218
219
220
221
222 lang = mLangDetector.findLang(lang, text);
223
224 if (lang != null) {
225 mLangs.add(lang);
226 }
227 }
228
229 @Override
230 public void onEmptySectionFound(List<Leaf> leaves) {
231
232 }
233 }