1   package org.daisy.pipeline.file.calabash.impl;
2   
3   import java.io.BufferedReader;
4   import java.io.InputStream;
5   import java.io.InputStreamReader;
6   import java.io.IOException;
7   import java.io.Reader;
8   import java.io.UnsupportedEncodingException;
9   import java.net.URI;
10  import java.net.URISyntaxException;
11  import java.nio.file.Path;
12  import java.nio.file.Files;
13  import java.util.Map;
14  
15  import com.xmlcalabash.core.XProcConstants;
16  import com.xmlcalabash.core.XProcRuntime;
17  import com.xmlcalabash.io.WritablePipe;
18  import com.xmlcalabash.library.DefaultStep;
19  import com.xmlcalabash.model.RuntimeValue;
20  import com.xmlcalabash.runtime.XAtomicStep;
21  import com.xmlcalabash.util.TreeWriter;
22  
23  import net.sf.saxon.s9api.QName;
24  import net.sf.saxon.s9api.SaxonApiException;
25  
26  import org.daisy.common.xproc.calabash.XProcStep;
27  import org.daisy.common.xproc.calabash.XProcStepProvider;
28  import org.daisy.common.xproc.XProcMonitor;
29  import org.daisy.pipeline.file.calabash.impl.PeekProvider.Peek;
30  import org.daisy.pipeline.file.FileUtils;
31  
32  import org.osgi.service.component.annotations.Component;
33  
34  @Component(
35  	name = "pxi:file-xml-peek",
36  	service = { XProcStepProvider.class },
37  	property = { "type:String={http://www.daisy.org/ns/pipeline/xproc/internal}file-xml-peek" }
38  )
39  public class XMLPeekProvider implements XProcStepProvider {
40  
41  	@Override
42  	public XProcStep newStep(XProcRuntime runtime, XAtomicStep step, XProcMonitor monitor, Map<String,String> properties) {
43  		return new XMLPeek(runtime, step);
44  	}
45  
46  	/**
47  	 * Returns the root element of the XML document referenced in `href`, including its attributes and namespaces.
48  	 */
49  	public static class XMLPeek extends DefaultStep implements XProcStep {
50  		private static final QName _href = new QName("href");
51  
52  		private WritablePipe result = null;
53  
54  		// XML bytes
55  		private byte byteLT = (byte) 60; // <
56  		private byte byteGT = (byte) 62; // >
57  		private byte byteQM = (byte) 63; // ?
58  		private byte byteEM = (byte) 33; // !
59  
60  		// BOM bytes
61  		private byte byte00 = (byte) 0;
62  		private byte byteBB = (byte) 187;
63  		private byte byteBF = (byte) 191;
64  		private byte byteEF = (byte) 239;
65  		private byte byteFE = (byte) 254;
66  		private byte byteFF = (byte) 255;
67  
68  		public XMLPeek(XProcRuntime runtime, XAtomicStep step) {
69  			super(runtime,step);
70  		}
71  
72  		public void setOutput(String port, WritablePipe pipe) {
73  			result = pipe;
74  		}
75  
76  		public void reset() {
77  			result.resetWriter();
78  		}
79  
80  		public void run() throws SaxonApiException {
81  			super.run();
82  
83  			RuntimeValue href = getOption(_href);
84  			URI sourceUri = href.getBaseURI().resolve(href.getString());
85  			Path file; {
86  				try {
87  					file = FileUtils.asPath(sourceUri);
88  				} catch (IllegalArgumentException|URISyntaxException e) {
89  					throw XProcStep.raiseError(new IllegalArgumentException("Illegal value for href option: " + href.getString(), e), step);
90  				}
91  			}
92  
93  			if (Files.isDirectory(file)) {
94  				throw XProcStep.raiseError(new IllegalArgumentException("Cannot peek into XML-file: file is a directory: " + file), step);
95  			}
96  
97  			String bomType = null;
98  			String bomBase64 = null;
99  			String bomHex = null;
100 			try {
101 				byte[] bomBytes = Peek.read(file, 0, 4);
102 
103 				if (bomBytes.length >= 3 && bomBytes[0] == byteEF && bomBytes[1] == byteBB && bomBytes[2] == byteBF) {
104 					logger.warn("File contains UTF-8 BOM:"+file);
105 					bomType = "UTF-8";
106 					bomBase64 = Peek.encodeBase64(spliceBytes(bomBytes, 0, 2));
107 					bomHex = "EF BB BF";
108 
109 				} else if (bomBytes.length >= 2 && bomBytes[0] == byteFE && bomBytes[1] == byteFF) {
110 					logger.warn("File contains UTF-16 (BE) BOM. Results may vary: "+file);
111 					bomType = "UTF-16 (BE)";
112 					bomBase64 = Peek.encodeBase64(spliceBytes(bomBytes, 0, 1));
113 					bomHex = "FE FF";
114 
115 				} else if (bomBytes.length >= 2 && bomBytes[0] == byteFF && bomBytes[1] == byteFE) {
116 					logger.warn("File contains UTF-16 (LE) BOM. Results may vary: "+file);
117 					bomType = "UTF-16 (LE)";
118 					bomBase64 = Peek.encodeBase64(spliceBytes(bomBytes, 0, 1));
119 					bomHex = "FF FE";
120 
121 				} else if (bomBytes.length >= 4 && bomBytes[0] == byte00 && bomBytes[1] == byte00 && bomBytes[2] == byteFE && bomBytes[3] == byteFF) {
122 					logger.warn("File contains UTF-32 (BE) BOM. Results may vary: "+file);
123 					bomType = "UTF-32 (BE)";
124 					bomBase64 = Peek.encodeBase64(spliceBytes(bomBytes, 0, 3));
125 					bomHex = "00 00 FE FF";
126 
127 				} else if (bomBytes.length >= 4 && bomBytes[0] == byteFF && bomBytes[1] == byteFE && bomBytes[2] == byte00 && bomBytes[3] == byte00) {
128 					logger.warn("File contains UTF-32 (LE) BOM. Results may vary: "+file);
129 					bomType = "UTF-32 (LE)";
130 					bomBase64 = Peek.encodeBase64(spliceBytes(bomBytes, 0, 3));
131 					bomHex = "FF FE 00 00";
132 				}
133 
134 			} catch (IOException | IndexOutOfBoundsException e) {
135 				logger.warn("px:file-xml-peek failed to read BOM from "+file, e);
136 			}
137 
138 			StringBuilder resultBuilder = new StringBuilder();
139 			Reader reader = null, buffer = null;
140 			InputStream in = null;
141 			try {
142 				in = Files.newInputStream(file);
143 				try {
144 					reader = new InputStreamReader(in, "UTF-8");
145 				} catch (UnsupportedEncodingException e) {
146 					reader = new InputStreamReader(in);
147 				}
148 				buffer = new BufferedReader(reader);
149 				int characterCount = 0;
150 				int r;
151 				char previousPreviousPreviousCharacter = ' ';
152 				char previousPreviousCharacter = ' ';
153 				char previousCharacter = ' ';
154 				boolean isReadingElement = false;
155 				boolean isReadingComment = false;
156 				char isReadingAttribute = ' ';
157 				while ((r = reader.read()) != -1) {
158 					// warn after 100K chars, break after 1M chars, we can possibly reduce these if this becomes a bottleneck
159 					characterCount++;
160 					if (characterCount == 100000) {
161 						logger.warn("More than 100 000 characters in prolog; this is probably not a XML file but will keep looking for root element a little while longer: "+file);
162 					}
163 					if (characterCount > 1000000) {
164 						logger.warn("More than 1 000 000 characters in prolog; this is extremely unlikely to be a XML file; will abort: "+file);
165 						resultBuilder = new StringBuilder(); // clear result contents
166 						break;
167 					}
168 
169 					char character = (char) r;
170 
171 					if (previousPreviousPreviousCharacter == '<' && previousPreviousCharacter == '!' && previousCharacter == '-' && character == '-') {
172 						isReadingComment = true;
173 						
174 					} else if (isReadingAttribute == ' ' && previousPreviousCharacter == '-' && previousCharacter == '-' && character == '>') {
175 						isReadingComment = false;
176 
177 					} else if (!isReadingComment && previousCharacter == '<' && (character != '!' && character != '?')) {
178 						isReadingElement = true;
179 						
180 					} else if (isReadingElement && (character == '\'' || character == '"') && (isReadingAttribute == character || isReadingAttribute == ' ')) {
181 						if (character != isReadingAttribute) {
182 							isReadingAttribute = character;
183 						} else {
184 							isReadingAttribute = ' ';
185 						}
186 					
187 					} else if (isReadingElement && isReadingAttribute == ' ' && character == '>') {
188 						if (previousCharacter != '/') {
189 							resultBuilder.append('/');
190 						}
191 						resultBuilder.append('>');
192 						break;
193 					}
194 
195 					previousPreviousPreviousCharacter = previousPreviousCharacter;
196 					previousPreviousCharacter = previousCharacter;
197 					previousCharacter = character;
198 					resultBuilder.append(character);
199 				}
200 			} catch (IOException e) {
201 				logger.error("px:file-xml-peek failed to read from "+file, e);
202 				e.printStackTrace();
203 
204 			} finally {
205 				if (buffer != null) {
206 					try {
207 						buffer.close();
208 					} catch (IOException e) {}
209 				}
210 				if (reader != null) {
211 					try {
212 						reader.close();
213 					} catch (IOException e) {}
214 				}
215 				if (in != null) {
216 					try {
217 						in.close();
218 					} catch (IOException e) {}
219 				}
220 			}
221 			String resultString = resultBuilder.toString();
222 
223 			TreeWriter tree = new TreeWriter(runtime);
224 			tree.startDocument(step.getNode().getBaseURI());
225 			tree.addStartElement(XProcConstants.c_result);
226 			tree.addAttribute(new QName("content-type"), "text/plain; charset=utf-8");
227 			if (bomType != null) {
228 				tree.addAttribute(new QName("bom-type"), bomType);
229 			}
230 			if (bomBase64 != null) {
231 				tree.addAttribute(new QName("bom-base64"), bomBase64);
232 			}
233 			if (bomBase64 != null) {
234 				tree.addAttribute(new QName("bom-hex"), bomHex);
235 			}
236 			tree.startContent();
237 
238 			tree.addText(resultString);
239 
240 			tree.addEndElement();
241 			tree.endDocument();
242 
243 			result.write(tree.getResult());
244 
245 		}
246 
247 		public static byte[] spliceBytes(byte[] bytes, int from, int to) {
248 			from = Math.max(0, from);
249 			to = Math.min(bytes.length-1, to);
250 			byte[] resultBytes = new byte[to - from + 1];
251 			for (int b = 0; b < resultBytes.length; b++) {
252 				resultBytes[b] = bytes[b + from];
253 			}
254 			return resultBytes;
255 		}
256 	}
257 
258 }