001package org.hl7.fhir.utilities.xhtml;
002
003import lombok.Getter;
004import lombok.Setter;
005import org.hl7.fhir.exceptions.FHIRException;
006import org.hl7.fhir.utilities.Utilities;
007
008import java.util.List;
009import java.util.ArrayList;
010
011/**
012 * Convert XHTML to GFM flavoured markdown
013 * <p>
014 * This isn't always possible, but we do our best. The primary issue is that
015 * GFM tables cannot be complex - if they contain nested tables, lists, or any other block content, the text format doesn't work
016 * There's no much we can do about that, really, so we kid of try to suppress paragraphs but if there is nested content, it'll collapse as
017 * a table.
018 * <p>
019 * Given that the primary target is AI systems this seems like a workable compromise
020 * <p>
021 * The filters are used by the IG publisher to strip out the common template content from the pages
022 */
023public class XhtmlToMarkdownConverter {
024
025  private boolean allowForms;
026
027  @Getter
028  private List<String> imageFilters = new ArrayList<>();
029  @Getter
030  private List<String> classFilters = new ArrayList<>();
031  @Getter
032  private List<String> idFilters = new ArrayList<>();
033  @Getter @Setter
034  private boolean ignoreGeneratedTables;
035  @Getter @Setter
036  private boolean aiMode;
037  @Getter @Setter
038  private boolean processUnknown;
039
040  public XhtmlToMarkdownConverter(boolean allowForms) {
041    this.allowForms = allowForms;
042  }
043
044  public String convert(XhtmlNode x) {
045    StringBuilder b = new StringBuilder();
046    convert(b, x, true);
047    return b.toString();
048  }
049
050  private void convert(StringBuilder b, XhtmlNode x, boolean allowParagraphs) {
051    for (XhtmlNode c : x.getChildNodes()) {
052      if (c.getNodeType() == NodeType.Text) {
053        paragraph(b, c, true, true, true);
054      } else if (c.getName() != null) {
055        convertNode(b, c, allowParagraphs);
056      }
057    }
058
059  }
060
061  public void convertNode(StringBuilder b, XhtmlNode c, boolean allowParagraphs) throws Error {
062    if (!elementPasses(c)) {
063      return;
064    }
065    if (allowParagraphs) {
066      para(b);
067    }
068    switch (c.getName()) {
069      case "p":
070        paragraph(b, c, true, false, allowParagraphs);
071        break;
072      case "hr":
073        b.append("-------\r\n");
074        break;
075      case "a":
076        paragraph(b, c, true, false, allowParagraphs);
077        break;
078      case "h1":
079        b.append("# " + toMDText(c.allText()));
080        if (allowParagraphs) {para(b); }
081        break;
082      case "h2":
083        b.append("## " + toMDText(c.allText()));
084        if (allowParagraphs) {para(b); }
085        break;
086      case "h3":
087        b.append("### " + toMDText(c.allText()));
088        if (allowParagraphs) {para(b); }
089        break;
090      case "h4":
091        b.append("#### " + toMDText(c.allText()));
092        if (allowParagraphs) {para(b); }
093        break;
094      case "h5":
095        b.append("##### " + toMDText(c.allText()));
096        if (allowParagraphs) {para(b); }
097        break;
098      case "h6":
099        b.append("###### " + toMDText(c.allText()));
100        if (allowParagraphs) {para(b); }
101        break;
102      case "strong":
103      case "em":
104      case "i":
105      case "span":
106      case "img":
107      case "br":
108      case "wbr":
109      case "del":
110      case "sup":
111      case "sub":
112      case "u":
113      case "b":
114        paragraph(b, c, true, false, allowParagraphs);
115        break;
116      case "code":
117        paragraph(b, c, true, false, allowParagraphs);
118        break;
119      case "div":
120        convert(b, c, allowParagraphs);
121        if (allowParagraphs) para(b);
122        break;
123      case "ul":
124        convertUL(b, c);
125        break;
126      case "li":
127        convertLI(b, c);
128        break;
129      case "ol":
130        convertOL(b, c);
131        break;
132      case "table":
133      case "tbody":
134        convertTable(b, c);
135        break;
136      case "iframe":
137        convertIFrame(b, c);
138        break;
139      case "form":
140      case "select":
141      case "input":
142      case "textarea":
143      case "option":
144        if (!allowForms) {
145          convert(b, c, allowParagraphs);
146        } else {
147          convertForm(b, c);
148        }
149        break;
150      case "label":
151      case "cite":
152        convertLabel(b, c, allowParagraphs);
153        break;
154      case "figure":
155      case "figcaption":
156        convert(b, c, allowParagraphs);
157        break;
158      case "dl":
159      case "tt":
160        convertDL(b, c);
161        break;
162      case "blockquote":
163        b.append("> ");
164        paragraph(b, c, false, true, allowParagraphs);
165        b.append("\r\n");
166        break;
167      case "pre":
168        b.append("```\r\n");
169        XhtmlNode code = c.getElement("code");
170        if (code != null) {
171          b.append(code.toLiteralText());
172        } else {
173          b.append(c.toLiteralText());
174        }
175        b.append("\r\n```\r\n");
176        break;
177      case "para":
178        // in error....
179        convert(b, c, allowParagraphs);
180        break;
181      case "tr":
182      case "td":
183      case "th":
184        // badly formed tables are ignored
185        convert(b, c, allowParagraphs);
186        break;
187      case "title":
188        b.append("# " + toMDText(c.allText()));
189        if (allowParagraphs) {para(b); }
190        break;
191      case "style":
192      case "script":
193        break;
194      // these are ignored if we're in 'page' mode - we just process the content
195      case "html":
196      case "head":
197      case "meta":
198      case "link":
199      case "stylesheet":
200      case "button":
201      case "nav":
202      case "body":
203        if (allowForms) {
204          throw new Error("not done yet: " + c.getName());
205        } else {
206          convert(b, c, allowParagraphs);
207          break;
208        }
209      case "svg":
210      case "object": // for svgs?
211        // do nothing
212        break;
213      case "details":
214        break;
215      case "header":
216        break;
217      case "footer":
218        break;
219      case "embed":
220        break;
221      case "mark":
222        break;
223      case "map":
224        break;
225      case "center":
226      case "font":
227        convert(b, c, allowParagraphs);
228        break;
229      default:
230        if (processUnknown) {
231          convert(b, c, allowParagraphs);
232        } else {
233          throw new FHIRException("Illegal HTML element: " + c.getName());
234        }
235    }
236  }
237
238  private boolean elementPasses(XhtmlNode x) {
239    String clss = x.getAttribute("class");
240    if (clss != null) {
241      for (String f : classFilters) {
242        if (clss.matches(f) || clss.contains(f)) {
243          return false;
244        }
245      }
246    }
247    String id = x.getAttribute("id");
248    if (id != null) {
249      for (String f : idFilters) {
250        if (id.matches(f)) {
251          return false;
252        }
253      }
254    }
255    if (aiMode && x.hasAttribute("data-fhir")) {
256      return !x.getAttribute("data-fhir").startsWith("generated");
257    }
258    return true;
259  }
260
261  private void para(StringBuilder b) {
262    if (!atStartOfPara2(b)) {
263      b.append("\r\n");
264    }
265  }
266
267  private void convertDL(StringBuilder b, XhtmlNode c) {
268  }
269
270  private void convertIFrame(StringBuilder b, XhtmlNode c) {
271    b.append(c.toString());
272  }
273
274  private void convertForm(StringBuilder b, XhtmlNode c) {
275    b.append(c.toString());
276  }
277
278  private void convertLabel(StringBuilder b, XhtmlNode c, boolean allowParagraphs) {
279    convert(b, c, allowParagraphs);
280  }
281
282  /**
283   * we're using the GFM format for tables, which doesn't allow nesting (not even any lists in tables)
284   * we'll try to follow that, but if the table is complex, it won't work. Still, better to sort of get it right in markdown
285   *
286   * @param b
287   * @param x
288   */
289  private void convertTable(StringBuilder b, XhtmlNode x) {
290    if (ignoreGeneratedTables && x.hasAttribute("fhir") && x.getAttribute("fhir").startsWith("generated")) {
291      return;
292    }
293    para(b);
294    List<XhtmlNode> rows = new ArrayList<>();
295    XhtmlNode headerRow = null;
296    headerRow = processChildNodesForTable(rows, x, false);
297    if (headerRow != null && !cells(headerRow).isEmpty() && aiMode) {
298      List<String> headings = new ArrayList<>();
299      for (XhtmlNode cell : cells(headerRow)) {
300        StringBuilder bh = new StringBuilder();
301        paragraph(bh, cell, false, true, false);
302        String p = bh.toString().trim();
303        headings.add(p);
304      }
305      for (XhtmlNode row : rows) {
306        List<XhtmlNode> cells = cells(row);
307        if (!cells.isEmpty()) {
308          b.append("* "+headings.get(0) + ": ");
309          paragraph(b, cells.get(0), false, true, false);
310          b.append("\r\n");
311          for (int i = 1; i < cells.size(); i++) {
312            String h = i < headings.size() ? headings.get(i) : "?";
313            b.append("  * " + h + ": ");
314            paragraph(b, cells.get(i), false, true, false);
315            b.append("\r\n");
316          }
317        }
318      }
319      b.append("\r\n");
320    } else {
321
322      int colCount = cells(headerRow).size();
323      for (XhtmlNode row : rows) {
324        colCount = Integer.max(colCount, cells(row).size());
325      }
326      // header row always exists but might be empty
327      int c = 0;
328      if (headerRow != null) {
329        for (XhtmlNode cell : cells(headerRow)) {
330          b.append("| ");
331          paragraph(b, cell, false, true, false);
332          b.append(" ");
333          c++;
334        }
335      }
336      for (int i = c; i < colCount; i++) {
337        b.append("| ");
338      }
339      b.append("|\r\n");
340      for (int i = 0; i < colCount; i++) {
341        b.append("| :--- ");
342      }
343      b.append("|\r\n");
344      for (XhtmlNode row : rows) {
345        c = 0;
346        for (XhtmlNode cell : cells(row)) {
347          b.append("| ");
348          paragraph(b, cell, false, true, false);
349          b.append(" ");
350          c++;
351        }
352        for (int i = c; i < colCount; i++) {
353          b.append("| ");
354        }
355        b.append("|\r\n");
356      }
357    }
358  }
359
360  private List<XhtmlNode> cells(XhtmlNode row) {
361    List<XhtmlNode> cells = new ArrayList<>();
362    if (row != null) {
363      for (XhtmlNode child : row.getChildNodes()) {
364        if (Utilities.existsInList(child.getName(), "td", "th")) {
365          cells.add(child);
366        }
367      }
368    }
369    return cells;
370  }
371
372  private XhtmlNode processChildNodesForTable(List<XhtmlNode> rows, XhtmlNode x, boolean header) {
373    XhtmlNode headerRow = null;
374    for (XhtmlNode child : x.getChildNodes()) {
375      if (child.getName() != null) {
376        switch (child.getName()) {
377          case "tbody":
378            headerRow = processChildNodesForTable(rows, child, false);
379            break;
380          case "thead":
381          case "tfoot":
382            processChildNodesForTable(rows, child, false);
383            break;
384          case "tr":
385            if (headerRow == null && (header || hasTHs(child)) || hasBoldedTDs(child)) {
386              headerRow = child;
387            } else {
388              rows.add(child);
389            }
390            break;
391          default:
392            // ignore
393        }
394      }
395    }
396    return headerRow;
397  }
398
399  private boolean hasTHs(XhtmlNode x) {
400    for (XhtmlNode child : x.getChildNodes()) {
401      if ("th".equals(child.getName())) {
402        return true;
403      }
404    }
405    return false;
406  }
407
408  private boolean hasBoldedTDs(XhtmlNode x) {
409    for (XhtmlNode child : x.getChildNodes()) {
410      if ("td".equals(child.getName())) {
411        return child.firstNamedDescendent("b") != null;
412      }
413    }
414    return false;
415  }
416
417  private void convertUL(StringBuilder b, XhtmlNode x) {
418
419    for (XhtmlNode c : x.getChildNodes()) {
420      if ("li".equals(c.getName())) {
421        b.append("* ");
422        int count = c.countChildrenByName("p");
423        if (count == 0) {
424          paragraph(b, c, true, false, true);
425        } else if (count == 1) {
426          paragraph(b, c.firstNamedDescendent("p"), true, false, true);
427        } else { // count > 1
428          boolean first = true;
429          for (XhtmlNode g : c.getChildNodes()) {
430            if ("p".equals(g.getName())) {
431              if (first) {
432                first = false;
433              } else {
434                b.append("\r\n\r\n  ");
435              }
436              paragraph(b, g, true, false, true);
437            }
438          }
439        }
440      }
441    }
442  }
443
444  private void convertLI(StringBuilder b, XhtmlNode x) {
445    b.append("* ");
446    paragraph(b, x, true, false, true);
447  }
448
449  private void convertOL(StringBuilder b, XhtmlNode x) {
450
451    for (XhtmlNode c : x.getChildNodes()) {
452      if ("li".equals(c.getName())) {
453        b.append("1. ");
454        paragraph(b, c, true, true, true);
455      }
456    }
457  }
458
459  private void paragraph(StringBuilder b, XhtmlNode x, boolean mark, boolean trim, boolean allowParagraphs) {
460    if (atStartOfPara2(b) && Utilities.noString(x.allText().trim())) {
461      return;
462    }
463    if (x.getNodeType() == NodeType.Text) {
464      addText(b, trim, x.getContent());
465    } else if (x.getName() != null && !Utilities.existsInList(x.getName(), "p", "li", "td", "th", "blockquote")) {
466      part(b, x, trim, allowParagraphs, true);
467    } else {
468      for (XhtmlNode c : x.getChildNodes()) {
469        if (c.getNodeType() == NodeType.Text) {
470          addText(b, trim, c.getContent());
471        } else if (c.getName() != null) {
472          part(b, c, trim, allowParagraphs, true);
473        }
474      }
475      if (mark && allowParagraphs) {
476        b.append("\r\n");
477      }
478    }
479  }
480
481  private boolean atStartOfPara2(StringBuilder b) {
482    switch (b.length()) {
483      case 0:
484        return true;
485      case 2:
486        return "\r\n".equals(b.substring(b.length() - 2));
487      default:
488        if (b.length() >= 4) {
489          return "\r\n\r\n".equals(b.substring(b.length() - 4));
490        } else {
491          return false;
492        }
493    }
494  }
495
496  private void addText(StringBuilder b, boolean trim, String content) {
497    if (trim) {
498      content = content.trim();
499    }
500    if (!Utilities.noString(content)) {
501      b.append(toMDText(content));
502    }
503  }
504
505  private String toMDText(String input) {
506    if (input == null || input.isEmpty()) {
507      return input;
508    }
509
510    StringBuilder result = new StringBuilder(input.length());
511    boolean inWhitespace = false;
512
513    for (int i = 0; i < input.length(); i++) {
514      char c = input.charAt(i);
515
516      if (Character.isWhitespace(c)) {
517        if (!inWhitespace) {
518          result.append(' ');
519          inWhitespace = true;
520        }
521        // Skip subsequent whitespace characters
522      } else {
523        result.append(c);
524        inWhitespace = false;
525      }
526    }
527
528    return result.toString();
529  }
530
531  public void part(StringBuilder b, XhtmlNode c, boolean trim, boolean allowParagraphs, boolean inPara) throws Error {
532    if (!elementPasses(c)) {
533      return;
534    }
535
536    switch (c.getName()) {
537      case "a":
538        if (c.hasAttribute("href") && hasSomeChildren(c)) {
539          b.append("[");
540          process(b, c, trim, allowParagraphs, inPara);
541          b.append("](");
542          b.append(fixRef(c.getAttribute("href")));
543          b.append(")");
544        } else {
545          process(b, c, trim, allowParagraphs, inPara);
546        }
547        break;
548      case "img":
549        if (imageRefPasses(c.getAttribute("src"))) {
550          b.append("![");
551          b.append(c.allText());
552          b.append("](");
553          b.append(c.getAttribute("src"));
554          b.append(")");
555        }
556        break;
557      case "samp":
558        b.append("`");
559        process(b, c, trim, false, inPara);
560        b.append("`");
561        break;
562      case "code":
563        String ticks = inPara ? "`" : "```";
564        b.append(ticks);
565        process(b, c, trim, allowParagraphs, inPara);
566        b.append(ticks);
567        break;
568      case "pre":
569        b.append("```");
570        process(b, c, trim, allowParagraphs, inPara);
571        b.append("```");
572        break;
573      case "strong":
574      case "em":
575      case "b":
576      case "mark":
577        b.append("**");
578        process(b, c, trim, allowParagraphs, inPara);
579        b.append("**");
580        break;
581      case "u":
582        b.append("_");
583        process(b, c, trim, allowParagraphs, inPara);
584        b.append("_");
585        break;
586      case "i":
587        b.append("*");
588        process(b, c, trim, allowParagraphs, inPara);
589        b.append("*");
590        break;
591      case "span":
592      case "sup":
593      case "sub":
594      case "s":
595        process(b, c, trim, allowParagraphs, inPara);
596        break;
597      case "div":
598      case "title":
599      case "h1":
600      case "h2":
601      case "h3":
602      case "h4":
603      case "h5":
604      case "h6":
605      case "table":
606      case "cite":
607        if (allowParagraphs) para(b);
608        convertNode(b, c, allowParagraphs);
609        if (allowParagraphs) para(b);
610        break;
611      case "p":
612        if (!trim && allowParagraphs) {
613          para(b);
614        }
615        process(b, c, trim, allowParagraphs, inPara);
616        break;
617      case "ul":
618        if (allowParagraphs) para(b);
619        convertUL(b, c);
620        break;
621      case "li":
622        if (allowParagraphs) para(b);
623        convertLI(b, c);
624        break;
625      case "ol":
626        if (allowParagraphs) para(b);
627        convertOL(b, c);
628        break;
629      case "blockquote":
630        if (allowParagraphs) para(b);
631        b.append("> ");
632        paragraph(b, c, false, trim, allowParagraphs);
633        if (allowParagraphs) b.append("\r\n");
634        break;
635      case "hr":
636        if (allowParagraphs) para(b);
637        b.append("-------");
638        if (allowParagraphs) para(b);
639        break;
640      case "del":
641      case "dl":
642      case "dd":
643      case "dt":
644      case "tt":
645      case "font":
646        process(b, c, trim, allowParagraphs, inPara);
647        break;
648      case "tr": // badly formed tables are ignored
649      case "td":
650      case "th":
651        process(b, c, trim, allowParagraphs, inPara);
652        break;
653      case "br":
654        if (allowParagraphs) para(b);
655        process(b, c, trim, allowParagraphs, inPara);
656        break;
657      case "wbr":
658        para(b);
659        break;
660      case "button":
661      case "input":
662      case "select":
663        if (allowForms) {
664          throw new Error("not done yet: " + c.getName());
665        } else {
666          convert(b, c, allowParagraphs);
667          break;
668        }
669      case "svg":
670        // not clear what, if anything, we should do with svg
671        break;
672      case "object": case "embed":
673        // not clear what, if anything, we should do with object if it's an svg, else we ignore it
674        break;
675      case "script":
676        // nothing
677        break;
678      case "base":
679        // nothing
680        break;
681      default:
682        if (processUnknown) {
683          // for weird stuff found on hl7.org/fhir
684          if ("A".equals(c.getName())) {
685            if (c.hasAttribute("href") && hasSomeChildren(c)) {
686              b.append("[");
687              process(b, c, trim, allowParagraphs, inPara);
688              b.append("](");
689              b.append(fixRef(c.getAttribute("href")));
690              b.append(")");
691            } else {
692              process(b, c, trim, allowParagraphs, inPara);
693            }
694          } else {
695            process(b, c, trim, allowParagraphs, inPara);
696          }
697        } else {
698          throw new FHIRException("illegal html element: " + c.getName() + " (" + c.allText() + ")");
699        }
700    }
701  }
702
703  private String fixRef(String href) {
704    if (aiMode && !Utilities.isAbsoluteUrl(href)) {
705      return href.replace(".html", ".md");
706    } else {
707      return href;
708    }
709  }
710
711  private boolean hasSomeChildren(XhtmlNode c) {
712    for (XhtmlNode child : c.getChildNodes()) {
713      if ("img".equals(child.getName())) {
714        if (imageRefPasses(child.getAttribute("src"))) {
715          return false;
716        }
717      }
718      if (elementPasses(child)) {
719        if (hasSomeChildren(child)) {
720          return true;
721        }
722      }
723      if (child.getContent() != null && !Utilities.noString(child.getContent().trim())) {
724        return true;
725      }
726    }
727    return false;
728  }
729
730  private boolean imageRefPasses(String src) {
731    for (String f : imageFilters) {
732      if (src.matches(f)) {
733        return false;
734      }
735    }
736    return true;
737  }
738
739  private void process(StringBuilder b, XhtmlNode x, boolean trim, boolean allowParagraphs, boolean inPara) {
740    for (XhtmlNode c : x.getChildNodes()) {
741      if (c.getNodeType() == NodeType.Text) {
742        addText(b, true, c.getContent());
743      } else if (c.getName() != null) {
744        part(b, c, trim, allowParagraphs, inPara);
745      }
746    }
747  }
748
749  public void configureForIGs(boolean processUnknown) {
750    getImageFilters().add("^tbl_.*");
751    getImageFilters().add("^icon_.*");
752    getImageFilters().add("^assets\\/*");
753    getImageFilters().add("^https://hl7\\.org/.*");
754    getImageFilters().add("^tree-filter");
755    getIdFilters().add("^segment-header");
756    getIdFilters().add("^segment-navbar");
757    getIdFilters().add("^publish-box");
758    getIdFilters().add("^segment-footer");
759    getIdFilters().add("^ppprofile");
760    getClassFilters().add("nav-tabs");
761    getClassFilters().add("markdown-toc");
762    setIgnoreGeneratedTables(true);
763    setAiMode(true);
764    this.processUnknown = processUnknown;
765  }
766}