001package org.hl7.fhir.utilities.i18n; 002 003import java.io.*; 004import java.nio.charset.StandardCharsets; 005import java.util.ArrayList; 006import java.util.Collections; 007import java.util.Comparator; 008import java.util.HashMap; 009import java.util.HashSet; 010import java.util.List; 011import java.util.Map; 012import java.util.Set; 013 014import org.hl7.fhir.utilities.FileUtilities; 015import org.hl7.fhir.utilities.IniFile; 016import org.hl7.fhir.utilities.StringPair; 017import org.hl7.fhir.utilities.Utilities; 018import org.hl7.fhir.utilities.filesystem.DirectoryVisitor; 019import org.hl7.fhir.utilities.filesystem.DirectoryVisitor.IDirectoryVisitorImplementation; 020import org.hl7.fhir.utilities.filesystem.ManagedFileAccess; 021 022/** 023 * This class checks that all the i18n constants and declarations are consistent, 024 * and then generates / updates the .po source files, and then updates the .properties files 025 * <br/> 026 * It needs to be run whenever 027 * (a) New constants are added to the java code 028 * (b) An implementer contributes translations in a .po source file 029 * <br/> 030 * It takes 3 parameters: 031 * * path to the local copy of the core repo 032 * * path to the local copy of the ig-publisher repo 033 * * path to the local copy of the fhirserver repo 034 */ 035@SuppressWarnings("checkstyle:systemout") 036public class POGenerator { 037 038 public class PropertyValue extends StringPair { 039 private boolean used; 040 041 public PropertyValue(String name, String value) { 042 super(name, value); 043 // TODO Auto-generated constructor stub 044 } 045 046 public String getBaseName() { 047 String res = getName(); 048 if (res.endsWith("_one")) { 049 res = res.substring(0, res.length()-4); 050 } else if (res.endsWith("_other")) { 051 res = res.substring(0, res.length()-6); 052 } 053 return res; 054 } 055 056 } 057 058 public class ConstantDefinition { 059 private String name; 060 private String sname; 061 private boolean defined; 062 private boolean used; 063 } 064 065 public class POObjectSorter implements Comparator<POObject> { 066 067 @Override 068 public int compare(POObject o1, POObject o2) { 069 return o1.getId().compareTo(o2.getId()); 070 } 071 } 072 073 public static void main(String[] args) throws IOException { 074 new POGenerator().execute(args[0], args[1], args[2]); 075 } 076 077 private int noTrans = 0; 078 079 public void execute(String core, String igpub, String pascal) throws IOException { 080 String source = Utilities.path(core, "/org.hl7.fhir.utilities/src/main/resources"); 081 if (checkState(source, core, igpub, pascal)) { 082 IniFile ini = new IniFile(Utilities.path(source, "translations-control.ini")); 083 generate(source, "rendering-phrases.properties", "rendering-phrases-en.po", null, 2); 084 generate(source, "Messages.properties", "validator-messages-en.po", null, 2); 085 for (String name : ini.getProperties("languages").keySet()) { 086 generate(source, "rendering-phrases.properties", "rendering-phrases-"+name+".po", "rendering-phrases_"+name+".properties", ini.getIntegerProperty("languages", name)); 087 generate(source, "Messages.properties", "validator-messages-"+name+".po", "Messages_"+name+".properties", ini.getIntegerProperty("languages", name)); 088 } 089 090 System.out.println("Finished"); 091 } 092 } 093 094 private boolean checkState(String source, String core, String igpub, String pascal) throws IOException { 095 System.out.println("Checking..."); 096 List<PropertyValue> props = loadProperties(Utilities.path(source, "rendering-phrases.properties"), true); 097 List<ConstantDefinition> consts = loadConstants(Utilities.path(core, "/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/i18n/RenderingI18nContext.java")); 098 boolean ok = true; 099 for (ConstantDefinition cd : consts) { 100 boolean found = false; 101 for (PropertyValue p : props) { 102 String pn = p.getBaseName(); 103 if (pn.equals(cd.sname) || p.getName().equals(cd.sname)) { 104 found = true; 105 p.used = true; 106 } 107 } 108 cd.defined = found; 109 } 110 scanJavaSource(core, consts, "RenderingI18nContext", "RenderingContext"); 111 scanJavaSource(igpub, consts, "RenderingI18nContext", "RenderingContext"); 112 scanPascalSource(pascal, props); 113 114 Set<String> pns = new HashSet<>(); 115 for (PropertyValue p : props) { 116 if (!p.used) { 117 ok = false; 118 System.out.println("Error: PV "+p.getName()+ " provided but not used"); 119 } 120 if (!pns.contains(p.getName())) { 121 pns.add(p.getName()); 122 } else { 123 System.out.println("Error: PV "+p.getName()+ " duplicated"); 124 } 125 if (p.getValue().contains("\\n")) { 126 System.out.println("Error: PV "+p.getName()+ " has a \\n"); 127 } 128 } 129 130 for (ConstantDefinition cd : consts) { 131 if (!cd.defined && !cd.used) { 132 System.out.println("Error: "+cd.name+ " not defined or used"); 133 ok = false; 134 } else if (!cd.defined) { 135 ok = false; 136 System.out.println("Error: msg for "+cd.name+ " not found at "+cd.sname); 137 } else if (!cd.used) { 138 System.out.println("Warning: const "+cd.name+ " not used"); 139 // ok = false; 140 } 141 } 142 143 props = loadProperties(Utilities.path(source, "Messages.properties"), true); 144 consts = loadConstants(Utilities.path(core, "/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/i18n/I18nConstants.java")); 145 for (ConstantDefinition cd : consts) { 146 boolean found = false; 147 for (PropertyValue p : props) { 148 String pn = p.getBaseName(); 149 if (pn.equals(cd.sname) || p.getName().equals(cd.sname)) { 150 found = true; 151 p.used = true; 152 } 153 } 154 cd.defined = found; 155 } 156 157 scanJavaSource(core, consts, "I18nConstants"); 158 scanJavaSource(igpub, consts, "I18nConstants"); 159 scanPascalSource(pascal, props); 160 161 pns = new HashSet<>(); 162 for (PropertyValue p : props) { 163 if (!p.used) { 164 ok = false; 165 System.out.println("Error: PV "+p.getName()+ " provided but not used"); 166 } 167 if (!pns.contains(p.getName())) { 168 pns.add(p.getName()); 169 } else { 170 System.out.println("Error: PV "+p.getName()+ " duplicated"); 171 } 172 if (p.getValue().contains("\\n")) { 173 System.out.println("Error: PV "+p.getName()+ " has a \\n"); 174 } 175 176 } 177 178 for (ConstantDefinition cd : consts) { 179 if (!cd.defined && !cd.used) { 180 System.out.println("Error: "+cd.name+ " not defined or used"); 181 ok = false; 182 } else if (!cd.defined) { 183 ok = false; 184 System.out.println("Error: msg for "+cd.name+ " not found @ "+cd.sname); 185 } else if (!cd.used) { 186 System.out.println("Warning: const "+cd.name+ " not used"); 187 // ok = false; 188 } 189 } 190 if (ok) { 191 System.out.println("No Errors Found"); 192 } else { 193 System.out.println("Errors Found, so not continuing"); 194 } 195 196 return ok; 197 } 198 199 private class JavaScanner implements IDirectoryVisitorImplementation { 200 List<ConstantDefinition> consts; 201 List<String> names; 202 203 @Override 204 public boolean enterDirectory(File f) throws IOException { 205 return !Utilities.existsInList(f.getName(), "model", "formats"); 206 } 207 208 @Override 209 public boolean visitFile(File file) throws IOException { 210 String source = FileUtilities.fileToString(file); 211 for (ConstantDefinition cd : consts) { 212 if (!cd.used) { 213 boolean found = false; 214 for (String n : names) { 215 if (source.contains(n+"."+cd.name+",")) { 216 found = true; 217 } 218 if (source.contains(n+"."+cd.name+")")) { 219 found = true; 220 } 221 if (source.contains(n+"."+cd.name+" :")) { 222 found = true; 223 } 224 if (source.contains(n+"."+cd.name+";")) { 225 found = true; 226 } 227 } 228 if (found) { 229 cd.used = true; 230 } 231 } 232 } 233 return true; 234 } 235 } 236 237 private void scanJavaSource(String path, List<ConstantDefinition> consts, String... names) throws FileNotFoundException, IOException { 238 JavaScanner scanner = new JavaScanner(); 239 scanner.consts = consts; 240 scanner.names = new ArrayList<String>(); 241 for (String s : names) { 242 scanner.names.add(s); 243 } 244 DirectoryVisitor.visitDirectory(scanner, path, "java"); 245 } 246 247 private class PascalScanner implements IDirectoryVisitorImplementation { 248 private List<PropertyValue> defs; 249 250 @Override 251 public boolean enterDirectory(File directory) throws IOException { 252 return true; 253 } 254 255 @Override 256 public boolean visitFile(File file) throws IOException { 257 String source = FileUtilities.fileToString(file); 258 for (PropertyValue pv : defs) { 259 if (!pv.used) { 260 boolean found = false; 261 String pn = pv.getBaseName(); 262 if (source.contains("'"+pn+"'")) { 263 found = true; 264 } 265 if (found) { 266 pv.used = true; 267 } 268 } 269 } 270 return true; 271 } 272 } 273 274 private void scanPascalSource(String path, List<PropertyValue> defs) throws FileNotFoundException, IOException { 275 PascalScanner scanner = new PascalScanner(); 276 scanner.defs = defs; 277 DirectoryVisitor.visitDirectory(scanner, path, "pas"); 278 } 279 280 281 private List<ConstantDefinition> loadConstants(String path) throws FileNotFoundException, IOException { 282 List<ConstantDefinition> res = new ArrayList<POGenerator.ConstantDefinition>(); 283 for (String line : FileUtilities.fileToLines(path)) { 284 if (line.contains("public static final String") && !line.trim().startsWith("//")) { 285 int i = line.indexOf("public static final String") + "public static final String".length(); 286 String[] p = line.substring(i).split("\\="); 287 if (p.length == 2) { 288 String n = p[0].trim(); 289 String v = p[1].trim().replace("\"", "").replace(";", ""); 290 ConstantDefinition cd = new ConstantDefinition(); 291 cd.name = n; 292 cd.sname = v; 293 res.add(cd); 294 } 295 } 296 } 297 return res; 298 } 299 300 enum PluralMode { 301 NONE, ONE, OTHER 302 } 303 304 private void generate(String sourceDirectory, String propertiesFileName, String poFileName, String targetPropertiesFileName, int count) throws IOException { 305 // load the destination file 306 // load the source file 307 // update the destination object set for changes from the source file 308 // save the destination file 309 String poFilePath = Utilities.path(sourceDirectory, "source", poFileName); 310 POSource poObjects; 311 if (ManagedFileAccess.file(poFilePath).exists()) { 312 poObjects = POSource.loadPOFile(poFilePath); 313 } else { 314 poObjects = new POSource(); 315 } 316 List<PropertyValue> javaProperties = loadProperties(Utilities.path(sourceDirectory, propertiesFileName), false); 317 for (PropertyValue javaProperty : javaProperties) { 318 String name = javaProperty.getName(); 319 PluralMode pluralMode = PluralMode.NONE; 320 if (name.endsWith("_one")) { 321 pluralMode = PluralMode.ONE; 322 name = name.substring(0, name.length() - 4); 323 } else if (name.endsWith("_other")) { 324 pluralMode = PluralMode.OTHER; 325 name = name.substring(0, name.length() - 6); 326 } 327 328 POObject poObject = findObject(poObjects.getPOObjects(), name); 329 if (poObject == null) { 330 if (pluralMode == PluralMode.OTHER) { 331 throw new Error("Not right"); 332 } 333 poObject = new POObject(); 334 poObject.setId(name); 335 poObject.setComment(name); 336 poObjects.getPOObjects().add(poObject); 337 poObject.setMsgid(javaProperty.getValue()); 338 poObject.setOrphan(false); 339 } else { 340 updatePOObject(poObject, pluralMode, javaProperty.getValue()); 341 } 342 } 343 poObjects.getPOObjects().removeIf(o -> o.isOrphan()); 344 Collections.sort(poObjects.getPOObjects(), new POObjectSorter()); 345 markDuplicatePOObjects(poObjects); 346 noTrans = poObjects.savePOFile(Utilities.path(sourceDirectory, "source", poFileName), count, noTrans); 347 if (targetPropertiesFileName != null) { 348 savePropertiesFile(Utilities.path(sourceDirectory, targetPropertiesFileName), poObjects.getPOObjects()); 349 } 350 } 351 352 private static void markDuplicatePOObjects(POSource poObjects) { 353 Map<String, Integer> sources = new HashMap<>(); 354 355 for (POObject poObject : poObjects.getPOObjects()) { 356 if (sources.containsKey(poObject.getMsgid())) { 357 Integer count = sources.get(poObject.getMsgid())+1; 358 sources.put(poObject.getMsgid(), count); 359// System.out.println("Duplicate in "+dest.substring(dest.lastIndexOf("/")+1)+": "+o.msgid+" on ("+CommaSeparatedStringBuilder.join(",", listIds(objects, o.msgid))+")"); 360 } else { 361 sources.put(poObject.getMsgid(), 1); 362 } 363 } 364 for (POObject poObject : poObjects.getPOObjects()) { 365 Integer count = sources.get(poObject.getMsgid()); 366 if (count > 1) { 367 poObject.setDuplicate(true); 368 } 369 } 370 } 371 372 private void updatePOObject(POObject poObject, PluralMode mode, String value) { 373 poObject.setOrphan(false); 374 if (poObject.getComment() == null) { 375 poObject.setComment(poObject.getId()); 376 } 377 if (mode == PluralMode.NONE) { 378 if (!value.equals(poObject.getMsgid())) { 379 // the english string has changed, and the other language string is now out of date 380 if (poObject.getOldMsgId() != null && !poObject.getMsgstr().isEmpty()) { 381 poObject.setOldMsgId(poObject.getMsgid()); 382 } 383 poObject.setMsgid(value); 384 for (int i = 0; i < poObject.getMsgstr().size(); i++) { 385 if (!Utilities.noString(poObject.getMsgstr().get(i))) { 386 poObject.getMsgstr().set(i, "!!"+poObject.getMsgstr().get(i)); 387 } 388 } 389 } else { 390 poObject.setOldMsgId(null); 391 } 392 } else if (mode == PluralMode.ONE) { 393 if (!value.equals(poObject.getMsgid())) { 394 // the english string has changed, and the other language string is now out of date 395 if (poObject.getOldMsgId() != null && !poObject.getMsgstr().isEmpty()) { 396 poObject.setOldMsgId(poObject.getMsgid()); 397 } 398 poObject.setMsgid(value); 399 if (poObject.getMsgstr().size() > 0 && !Utilities.noString(poObject.getMsgstr().get(0))) { 400 poObject.getMsgstr().set(0, "!!"+poObject.getMsgstr().get(0)); 401 } 402 } else { 403 poObject.setOldMsgId(null); 404 } 405 } else if (mode == PluralMode.OTHER) { 406 if (!value.equals(poObject.getMsgidPlural())) { 407 // the english string has changed, and the other language string is now out of date 408// if (o.oldMsgId != null) { 409// o.oldMsgId = o.msgid; 410// } 411 poObject.setMsgidPlural(value); 412 if (poObject.getMsgstr().size() > 1 && !Utilities.noString(poObject.getMsgstr().get(1))) { 413 poObject.getMsgstr().set(1, "!!"+poObject.getMsgstr().get(1)); 414 } 415 } else { 416 poObject.setOldMsgId(null); 417 } 418 } 419 } 420 421 private POObject findObject(List<POObject> poObjects, String name) { 422 for (POObject poObject : poObjects) { 423 if (poObject.getId() != null && poObject.getId().equals(name)) { 424 return poObject; 425 } 426 } 427 return null; 428 } 429 430 private List<PropertyValue> loadProperties(String source, boolean checking) throws IOException { 431 List<PropertyValue> res = new ArrayList<>(); 432 File src = ManagedFileAccess.file(source); 433 BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(src), StandardCharsets.UTF_8)); 434 for (String line = reader.readLine(); line != null; line = reader.readLine()) { 435 if (!line.startsWith("#") && line.contains("=")) { 436 String n = line.substring(0, line.indexOf("=")).trim(); 437 String v = line.substring(line.indexOf("=")+1).trim(); 438 if (checking || !(v.length() == 3 && v.startsWith("{") && v.endsWith("}"))) { 439 res.add(new PropertyValue(n, v)); 440 } 441 } 442 } 443 return res; 444 } 445 446 private void savePropertiesFile(String tgt, List<POObject> objects) throws IOException { 447 String nameLine = FileUtilities.fileToLines(tgt).get(0); 448 String[] parts = nameLine.substring(1).trim().split("\\="); 449 String[] names = parts[1].split("\\,"); 450 451 StringBuilder b = new StringBuilder(); 452 b.append(nameLine+"\r\n"); 453 for (POObject o : objects) { 454 if (o.getMsgidPlural() == null) { 455 String v= o.getMsgstr().size() > 0 ? o.getMsgstr().get(0) : ""; 456 if (!Utilities.noString(v)) { 457 if (v.startsWith("!!")) { 458 v = v.substring(2); 459 } 460 b.append(o.getId()+" = "+v+"\r\n"); 461 } 462 } else { 463 for (int i = 0; i < names.length; i++) { 464 String v = (o.getMsgstr().size() > i ? o.getMsgstr().get(i) : ""); 465 if (!Utilities.noString(v)) { 466 if (v.startsWith("!!")) { 467 v = v.substring(2); 468 } 469 b.append(o.getId()+"_"+names[i].trim()+" = "+v+"\r\n"); 470 } 471 } 472 } 473 } 474 475 FileUtilities.stringToFile(b.toString(), tgt); 476 } 477 478 479}