001package org.hl7.fhir.utilities.xhtml; 002 003import lombok.Getter; 004import lombok.Setter; 005import org.hl7.fhir.exceptions.FHIRException; 006import org.hl7.fhir.utilities.Utilities; 007 008import java.util.List; 009import java.util.ArrayList; 010 011/** 012 * Convert XHTML to GFM flavoured markdown 013 * <p> 014 * This isn't always possible, but we do our best. The primary issue is that 015 * GFM tables cannot be complex - if they contain nested tables, lists, or any other block content, the text format doesn't work 016 * There's no much we can do about that, really, so we kid of try to suppress paragraphs but if there is nested content, it'll collapse as 017 * a table. 018 * <p> 019 * Given that the primary target is AI systems this seems like a workable compromise 020 * <p> 021 * The filters are used by the IG publisher to strip out the common template content from the pages 022 */ 023public class XhtmlToMarkdownConverter { 024 025 private boolean allowForms; 026 027 @Getter 028 private List<String> imageFilters = new ArrayList<>(); 029 @Getter 030 private List<String> classFilters = new ArrayList<>(); 031 @Getter 032 private List<String> idFilters = new ArrayList<>(); 033 @Getter @Setter 034 private boolean ignoreGeneratedTables; 035 @Getter @Setter 036 private boolean aiMode; 037 @Getter @Setter 038 private boolean processUnknown; 039 040 public XhtmlToMarkdownConverter(boolean allowForms) { 041 this.allowForms = allowForms; 042 } 043 044 public String convert(XhtmlNode x) { 045 StringBuilder b = new StringBuilder(); 046 convert(b, x, true); 047 return b.toString(); 048 } 049 050 private void convert(StringBuilder b, XhtmlNode x, boolean allowParagraphs) { 051 for (XhtmlNode c : x.getChildNodes()) { 052 if (c.getNodeType() == NodeType.Text) { 053 paragraph(b, c, true, true, true); 054 } else if (c.getName() != null) { 055 convertNode(b, c, allowParagraphs); 056 } 057 } 058 059 } 060 061 public void convertNode(StringBuilder b, XhtmlNode c, boolean allowParagraphs) throws Error { 062 if (!elementPasses(c)) { 063 return; 064 } 065 if (allowParagraphs) { 066 para(b); 067 } 068 switch (c.getName()) { 069 case "p": 070 paragraph(b, c, true, false, allowParagraphs); 071 break; 072 case "hr": 073 b.append("-------\r\n"); 074 break; 075 case "a": 076 paragraph(b, c, true, false, allowParagraphs); 077 break; 078 case "h1": 079 b.append("# " + toMDText(c.allText())); 080 if (allowParagraphs) {para(b); } 081 break; 082 case "h2": 083 b.append("## " + toMDText(c.allText())); 084 if (allowParagraphs) {para(b); } 085 break; 086 case "h3": 087 b.append("### " + toMDText(c.allText())); 088 if (allowParagraphs) {para(b); } 089 break; 090 case "h4": 091 b.append("#### " + toMDText(c.allText())); 092 if (allowParagraphs) {para(b); } 093 break; 094 case "h5": 095 b.append("##### " + toMDText(c.allText())); 096 if (allowParagraphs) {para(b); } 097 break; 098 case "h6": 099 b.append("###### " + toMDText(c.allText())); 100 if (allowParagraphs) {para(b); } 101 break; 102 case "strong": 103 case "em": 104 case "i": 105 case "span": 106 case "img": 107 case "br": 108 case "wbr": 109 case "del": 110 case "sup": 111 case "sub": 112 case "u": 113 case "b": 114 paragraph(b, c, true, false, allowParagraphs); 115 break; 116 case "code": 117 paragraph(b, c, true, false, allowParagraphs); 118 break; 119 case "div": 120 convert(b, c, allowParagraphs); 121 if (allowParagraphs) para(b); 122 break; 123 case "ul": 124 convertUL(b, c); 125 break; 126 case "li": 127 convertLI(b, c); 128 break; 129 case "ol": 130 convertOL(b, c); 131 break; 132 case "table": 133 case "tbody": 134 convertTable(b, c); 135 break; 136 case "iframe": 137 convertIFrame(b, c); 138 break; 139 case "form": 140 case "select": 141 case "input": 142 case "textarea": 143 case "option": 144 if (!allowForms) { 145 convert(b, c, allowParagraphs); 146 } else { 147 convertForm(b, c); 148 } 149 break; 150 case "label": 151 case "cite": 152 convertLabel(b, c, allowParagraphs); 153 break; 154 case "figure": 155 case "figcaption": 156 convert(b, c, allowParagraphs); 157 break; 158 case "dl": 159 case "tt": 160 convertDL(b, c); 161 break; 162 case "blockquote": 163 b.append("> "); 164 paragraph(b, c, false, true, allowParagraphs); 165 b.append("\r\n"); 166 break; 167 case "pre": 168 b.append("```\r\n"); 169 XhtmlNode code = c.getElement("code"); 170 if (code != null) { 171 b.append(code.toLiteralText()); 172 } else { 173 b.append(c.toLiteralText()); 174 } 175 b.append("\r\n```\r\n"); 176 break; 177 case "para": 178 // in error.... 179 convert(b, c, allowParagraphs); 180 break; 181 case "tr": 182 case "td": 183 case "th": 184 // badly formed tables are ignored 185 convert(b, c, allowParagraphs); 186 break; 187 case "title": 188 b.append("# " + toMDText(c.allText())); 189 if (allowParagraphs) {para(b); } 190 break; 191 case "style": 192 case "script": 193 break; 194 // these are ignored if we're in 'page' mode - we just process the content 195 case "html": 196 case "head": 197 case "meta": 198 case "link": 199 case "stylesheet": 200 case "button": 201 case "nav": 202 case "body": 203 if (allowForms) { 204 throw new Error("not done yet: " + c.getName()); 205 } else { 206 convert(b, c, allowParagraphs); 207 break; 208 } 209 case "svg": 210 case "object": // for svgs? 211 // do nothing 212 break; 213 case "details": 214 break; 215 case "header": 216 break; 217 case "footer": 218 break; 219 case "embed": 220 break; 221 case "mark": 222 break; 223 case "map": 224 break; 225 case "center": 226 case "font": 227 convert(b, c, allowParagraphs); 228 break; 229 default: 230 if (processUnknown) { 231 convert(b, c, allowParagraphs); 232 } else { 233 throw new FHIRException("Illegal HTML element: " + c.getName()); 234 } 235 } 236 } 237 238 private boolean elementPasses(XhtmlNode x) { 239 String clss = x.getAttribute("class"); 240 if (clss != null) { 241 for (String f : classFilters) { 242 if (clss.matches(f) || clss.contains(f)) { 243 return false; 244 } 245 } 246 } 247 String id = x.getAttribute("id"); 248 if (id != null) { 249 for (String f : idFilters) { 250 if (id.matches(f)) { 251 return false; 252 } 253 } 254 } 255 if (aiMode && x.hasAttribute("data-fhir")) { 256 return !x.getAttribute("data-fhir").startsWith("generated"); 257 } 258 return true; 259 } 260 261 private void para(StringBuilder b) { 262 if (!atStartOfPara2(b)) { 263 b.append("\r\n"); 264 } 265 } 266 267 private void convertDL(StringBuilder b, XhtmlNode c) { 268 } 269 270 private void convertIFrame(StringBuilder b, XhtmlNode c) { 271 b.append(c.toString()); 272 } 273 274 private void convertForm(StringBuilder b, XhtmlNode c) { 275 b.append(c.toString()); 276 } 277 278 private void convertLabel(StringBuilder b, XhtmlNode c, boolean allowParagraphs) { 279 convert(b, c, allowParagraphs); 280 } 281 282 /** 283 * we're using the GFM format for tables, which doesn't allow nesting (not even any lists in tables) 284 * we'll try to follow that, but if the table is complex, it won't work. Still, better to sort of get it right in markdown 285 * 286 * @param b 287 * @param x 288 */ 289 private void convertTable(StringBuilder b, XhtmlNode x) { 290 if (ignoreGeneratedTables && x.hasAttribute("fhir") && x.getAttribute("fhir").startsWith("generated")) { 291 return; 292 } 293 para(b); 294 List<XhtmlNode> rows = new ArrayList<>(); 295 XhtmlNode headerRow = null; 296 headerRow = processChildNodesForTable(rows, x, false); 297 if (headerRow != null && !cells(headerRow).isEmpty() && aiMode) { 298 List<String> headings = new ArrayList<>(); 299 for (XhtmlNode cell : cells(headerRow)) { 300 StringBuilder bh = new StringBuilder(); 301 paragraph(bh, cell, false, true, false); 302 String p = bh.toString().trim(); 303 headings.add(p); 304 } 305 for (XhtmlNode row : rows) { 306 List<XhtmlNode> cells = cells(row); 307 if (!cells.isEmpty()) { 308 b.append("* "+headings.get(0) + ": "); 309 paragraph(b, cells.get(0), false, true, false); 310 b.append("\r\n"); 311 for (int i = 1; i < cells.size(); i++) { 312 String h = i < headings.size() ? headings.get(i) : "?"; 313 b.append(" * " + h + ": "); 314 paragraph(b, cells.get(i), false, true, false); 315 b.append("\r\n"); 316 } 317 } 318 } 319 b.append("\r\n"); 320 } else { 321 322 int colCount = cells(headerRow).size(); 323 for (XhtmlNode row : rows) { 324 colCount = Integer.max(colCount, cells(row).size()); 325 } 326 // header row always exists but might be empty 327 int c = 0; 328 if (headerRow != null) { 329 for (XhtmlNode cell : cells(headerRow)) { 330 b.append("| "); 331 paragraph(b, cell, false, true, false); 332 b.append(" "); 333 c++; 334 } 335 } 336 for (int i = c; i < colCount; i++) { 337 b.append("| "); 338 } 339 b.append("|\r\n"); 340 for (int i = 0; i < colCount; i++) { 341 b.append("| :--- "); 342 } 343 b.append("|\r\n"); 344 for (XhtmlNode row : rows) { 345 c = 0; 346 for (XhtmlNode cell : cells(row)) { 347 b.append("| "); 348 paragraph(b, cell, false, true, false); 349 b.append(" "); 350 c++; 351 } 352 for (int i = c; i < colCount; i++) { 353 b.append("| "); 354 } 355 b.append("|\r\n"); 356 } 357 } 358 } 359 360 private List<XhtmlNode> cells(XhtmlNode row) { 361 List<XhtmlNode> cells = new ArrayList<>(); 362 if (row != null) { 363 for (XhtmlNode child : row.getChildNodes()) { 364 if (Utilities.existsInList(child.getName(), "td", "th")) { 365 cells.add(child); 366 } 367 } 368 } 369 return cells; 370 } 371 372 private XhtmlNode processChildNodesForTable(List<XhtmlNode> rows, XhtmlNode x, boolean header) { 373 XhtmlNode headerRow = null; 374 for (XhtmlNode child : x.getChildNodes()) { 375 if (child.getName() != null) { 376 switch (child.getName()) { 377 case "tbody": 378 headerRow = processChildNodesForTable(rows, child, false); 379 break; 380 case "thead": 381 case "tfoot": 382 processChildNodesForTable(rows, child, false); 383 break; 384 case "tr": 385 if (headerRow == null && (header || hasTHs(child)) || hasBoldedTDs(child)) { 386 headerRow = child; 387 } else { 388 rows.add(child); 389 } 390 break; 391 default: 392 // ignore 393 } 394 } 395 } 396 return headerRow; 397 } 398 399 private boolean hasTHs(XhtmlNode x) { 400 for (XhtmlNode child : x.getChildNodes()) { 401 if ("th".equals(child.getName())) { 402 return true; 403 } 404 } 405 return false; 406 } 407 408 private boolean hasBoldedTDs(XhtmlNode x) { 409 for (XhtmlNode child : x.getChildNodes()) { 410 if ("td".equals(child.getName())) { 411 return child.firstNamedDescendent("b") != null; 412 } 413 } 414 return false; 415 } 416 417 private void convertUL(StringBuilder b, XhtmlNode x) { 418 419 for (XhtmlNode c : x.getChildNodes()) { 420 if ("li".equals(c.getName())) { 421 b.append("* "); 422 int count = c.countChildrenByName("p"); 423 if (count == 0) { 424 paragraph(b, c, true, false, true); 425 } else if (count == 1) { 426 paragraph(b, c.firstNamedDescendent("p"), true, false, true); 427 } else { // count > 1 428 boolean first = true; 429 for (XhtmlNode g : c.getChildNodes()) { 430 if ("p".equals(g.getName())) { 431 if (first) { 432 first = false; 433 } else { 434 b.append("\r\n\r\n "); 435 } 436 paragraph(b, g, true, false, true); 437 } 438 } 439 } 440 } 441 } 442 } 443 444 private void convertLI(StringBuilder b, XhtmlNode x) { 445 b.append("* "); 446 paragraph(b, x, true, false, true); 447 } 448 449 private void convertOL(StringBuilder b, XhtmlNode x) { 450 451 for (XhtmlNode c : x.getChildNodes()) { 452 if ("li".equals(c.getName())) { 453 b.append("1. "); 454 paragraph(b, c, true, true, true); 455 } 456 } 457 } 458 459 private void paragraph(StringBuilder b, XhtmlNode x, boolean mark, boolean trim, boolean allowParagraphs) { 460 if (atStartOfPara2(b) && Utilities.noString(x.allText().trim())) { 461 return; 462 } 463 if (x.getNodeType() == NodeType.Text) { 464 addText(b, trim, x.getContent()); 465 } else if (x.getName() != null && !Utilities.existsInList(x.getName(), "p", "li", "td", "th", "blockquote")) { 466 part(b, x, trim, allowParagraphs, true); 467 } else { 468 for (XhtmlNode c : x.getChildNodes()) { 469 if (c.getNodeType() == NodeType.Text) { 470 addText(b, trim, c.getContent()); 471 } else if (c.getName() != null) { 472 part(b, c, trim, allowParagraphs, true); 473 } 474 } 475 if (mark && allowParagraphs) { 476 b.append("\r\n"); 477 } 478 } 479 } 480 481 private boolean atStartOfPara2(StringBuilder b) { 482 switch (b.length()) { 483 case 0: 484 return true; 485 case 2: 486 return "\r\n".equals(b.substring(b.length() - 2)); 487 default: 488 if (b.length() >= 4) { 489 return "\r\n\r\n".equals(b.substring(b.length() - 4)); 490 } else { 491 return false; 492 } 493 } 494 } 495 496 private void addText(StringBuilder b, boolean trim, String content) { 497 if (trim) { 498 content = content.trim(); 499 } 500 if (!Utilities.noString(content)) { 501 b.append(toMDText(content)); 502 } 503 } 504 505 private String toMDText(String input) { 506 if (input == null || input.isEmpty()) { 507 return input; 508 } 509 510 StringBuilder result = new StringBuilder(input.length()); 511 boolean inWhitespace = false; 512 513 for (int i = 0; i < input.length(); i++) { 514 char c = input.charAt(i); 515 516 if (Character.isWhitespace(c)) { 517 if (!inWhitespace) { 518 result.append(' '); 519 inWhitespace = true; 520 } 521 // Skip subsequent whitespace characters 522 } else { 523 result.append(c); 524 inWhitespace = false; 525 } 526 } 527 528 return result.toString(); 529 } 530 531 public void part(StringBuilder b, XhtmlNode c, boolean trim, boolean allowParagraphs, boolean inPara) throws Error { 532 if (!elementPasses(c)) { 533 return; 534 } 535 536 switch (c.getName()) { 537 case "a": 538 if (c.hasAttribute("href") && hasSomeChildren(c)) { 539 b.append("["); 540 process(b, c, trim, allowParagraphs, inPara); 541 b.append("]("); 542 b.append(fixRef(c.getAttribute("href"))); 543 b.append(")"); 544 } else { 545 process(b, c, trim, allowParagraphs, inPara); 546 } 547 break; 548 case "img": 549 if (imageRefPasses(c.getAttribute("src"))) { 550 b.append("; 553 b.append(c.getAttribute("src")); 554 b.append(")"); 555 } 556 break; 557 case "samp": 558 b.append("`"); 559 process(b, c, trim, false, inPara); 560 b.append("`"); 561 break; 562 case "code": 563 String ticks = inPara ? "`" : "```"; 564 b.append(ticks); 565 process(b, c, trim, allowParagraphs, inPara); 566 b.append(ticks); 567 break; 568 case "pre": 569 b.append("```"); 570 process(b, c, trim, allowParagraphs, inPara); 571 b.append("```"); 572 break; 573 case "strong": 574 case "em": 575 case "b": 576 case "mark": 577 b.append("**"); 578 process(b, c, trim, allowParagraphs, inPara); 579 b.append("**"); 580 break; 581 case "u": 582 b.append("_"); 583 process(b, c, trim, allowParagraphs, inPara); 584 b.append("_"); 585 break; 586 case "i": 587 b.append("*"); 588 process(b, c, trim, allowParagraphs, inPara); 589 b.append("*"); 590 break; 591 case "span": 592 case "sup": 593 case "sub": 594 case "s": 595 process(b, c, trim, allowParagraphs, inPara); 596 break; 597 case "div": 598 case "title": 599 case "h1": 600 case "h2": 601 case "h3": 602 case "h4": 603 case "h5": 604 case "h6": 605 case "table": 606 case "cite": 607 if (allowParagraphs) para(b); 608 convertNode(b, c, allowParagraphs); 609 if (allowParagraphs) para(b); 610 break; 611 case "p": 612 if (!trim && allowParagraphs) { 613 para(b); 614 } 615 process(b, c, trim, allowParagraphs, inPara); 616 break; 617 case "ul": 618 if (allowParagraphs) para(b); 619 convertUL(b, c); 620 break; 621 case "li": 622 if (allowParagraphs) para(b); 623 convertLI(b, c); 624 break; 625 case "ol": 626 if (allowParagraphs) para(b); 627 convertOL(b, c); 628 break; 629 case "blockquote": 630 if (allowParagraphs) para(b); 631 b.append("> "); 632 paragraph(b, c, false, trim, allowParagraphs); 633 if (allowParagraphs) b.append("\r\n"); 634 break; 635 case "hr": 636 if (allowParagraphs) para(b); 637 b.append("-------"); 638 if (allowParagraphs) para(b); 639 break; 640 case "del": 641 case "dl": 642 case "dd": 643 case "dt": 644 case "tt": 645 case "font": 646 process(b, c, trim, allowParagraphs, inPara); 647 break; 648 case "tr": // badly formed tables are ignored 649 case "td": 650 case "th": 651 process(b, c, trim, allowParagraphs, inPara); 652 break; 653 case "br": 654 if (allowParagraphs) para(b); 655 process(b, c, trim, allowParagraphs, inPara); 656 break; 657 case "wbr": 658 para(b); 659 break; 660 case "button": 661 case "input": 662 case "select": 663 if (allowForms) { 664 throw new Error("not done yet: " + c.getName()); 665 } else { 666 convert(b, c, allowParagraphs); 667 break; 668 } 669 case "svg": 670 // not clear what, if anything, we should do with svg 671 break; 672 case "object": case "embed": 673 // not clear what, if anything, we should do with object if it's an svg, else we ignore it 674 break; 675 case "script": 676 // nothing 677 break; 678 case "base": 679 // nothing 680 break; 681 default: 682 if (processUnknown) { 683 // for weird stuff found on hl7.org/fhir 684 if ("A".equals(c.getName())) { 685 if (c.hasAttribute("href") && hasSomeChildren(c)) { 686 b.append("["); 687 process(b, c, trim, allowParagraphs, inPara); 688 b.append("]("); 689 b.append(fixRef(c.getAttribute("href"))); 690 b.append(")"); 691 } else { 692 process(b, c, trim, allowParagraphs, inPara); 693 } 694 } else { 695 process(b, c, trim, allowParagraphs, inPara); 696 } 697 } else { 698 throw new FHIRException("illegal html element: " + c.getName() + " (" + c.allText() + ")"); 699 } 700 } 701 } 702 703 private String fixRef(String href) { 704 if (aiMode && !Utilities.isAbsoluteUrl(href)) { 705 return href.replace(".html", ".md"); 706 } else { 707 return href; 708 } 709 } 710 711 private boolean hasSomeChildren(XhtmlNode c) { 712 for (XhtmlNode child : c.getChildNodes()) { 713 if ("img".equals(child.getName())) { 714 if (imageRefPasses(child.getAttribute("src"))) { 715 return false; 716 } 717 } 718 if (elementPasses(child)) { 719 if (hasSomeChildren(child)) { 720 return true; 721 } 722 } 723 if (child.getContent() != null && !Utilities.noString(child.getContent().trim())) { 724 return true; 725 } 726 } 727 return false; 728 } 729 730 private boolean imageRefPasses(String src) { 731 for (String f : imageFilters) { 732 if (src.matches(f)) { 733 return false; 734 } 735 } 736 return true; 737 } 738 739 private void process(StringBuilder b, XhtmlNode x, boolean trim, boolean allowParagraphs, boolean inPara) { 740 for (XhtmlNode c : x.getChildNodes()) { 741 if (c.getNodeType() == NodeType.Text) { 742 addText(b, true, c.getContent()); 743 } else if (c.getName() != null) { 744 part(b, c, trim, allowParagraphs, inPara); 745 } 746 } 747 } 748 749 public void configureForIGs(boolean processUnknown) { 750 getImageFilters().add("^tbl_.*"); 751 getImageFilters().add("^icon_.*"); 752 getImageFilters().add("^assets\\/*"); 753 getImageFilters().add("^https://hl7\\.org/.*"); 754 getImageFilters().add("^tree-filter"); 755 getIdFilters().add("^segment-header"); 756 getIdFilters().add("^segment-navbar"); 757 getIdFilters().add("^publish-box"); 758 getIdFilters().add("^segment-footer"); 759 getIdFilters().add("^ppprofile"); 760 getClassFilters().add("nav-tabs"); 761 getClassFilters().add("markdown-toc"); 762 setIgnoreGeneratedTables(true); 763 setAiMode(true); 764 this.processUnknown = processUnknown; 765 } 766}