001package org.hl7.fhir.utilities.i18n;
002
003import java.io.*;
004import java.nio.charset.StandardCharsets;
005import java.util.ArrayList;
006import java.util.Collections;
007import java.util.Comparator;
008import java.util.HashMap;
009import java.util.HashSet;
010import java.util.List;
011import java.util.Map;
012import java.util.Set;
013
014import org.hl7.fhir.utilities.FileUtilities;
015import org.hl7.fhir.utilities.IniFile;
016import org.hl7.fhir.utilities.StringPair;
017import org.hl7.fhir.utilities.Utilities;
018import org.hl7.fhir.utilities.filesystem.DirectoryVisitor;
019import org.hl7.fhir.utilities.filesystem.DirectoryVisitor.IDirectoryVisitorImplementation;
020import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
021
022/**
023 * This class checks that all the i18n constants and declarations are consistent,
024 * and then generates / updates the .po source files, and then updates the .properties files
025 * <br/>
026 * It needs to be run whenever
027 *   (a) New constants are added to the java code
028 *   (b) An implementer contributes translations in a .po source file
029 * <br/>
030 * It takes 3 parameters:
031 *   * path to the local copy of the core repo
032 *   * path to the local copy of the ig-publisher repo
033 *   * path to the local copy of the fhirserver repo
034 */
035@SuppressWarnings("checkstyle:systemout")
036public class POGenerator {
037
038  public class PropertyValue extends StringPair {
039    private boolean used;
040    
041    public PropertyValue(String name, String value) {
042      super(name, value);
043      // TODO Auto-generated constructor stub
044    }
045
046    public String getBaseName() {
047      String res = getName();
048      if (res.endsWith("_one")) {
049        res = res.substring(0, res.length()-4);
050      } else if (res.endsWith("_other")) {
051        res = res.substring(0, res.length()-6);
052      }
053      return res;
054    }
055
056  }
057
058  public class ConstantDefinition {
059    private String name;
060    private String sname;
061    private boolean defined;
062    private boolean used;
063  }
064
065  public class POObjectSorter implements Comparator<POObject> {
066
067    @Override
068    public int compare(POObject o1, POObject o2) {
069      return o1.getId().compareTo(o2.getId());
070    }
071  }
072
073  public static void main(String[] args) throws IOException {
074    new POGenerator().execute(args[0], args[1], args[2]);
075  }
076
077  private int noTrans = 0;
078
079  public void execute(String core, String igpub, String pascal) throws IOException {
080    String source = Utilities.path(core, "/org.hl7.fhir.utilities/src/main/resources");
081    if (checkState(source, core, igpub, pascal)) {
082      IniFile ini = new IniFile(Utilities.path(source, "translations-control.ini"));
083      generate(source, "rendering-phrases.properties",  "rendering-phrases-en.po",       null, 2);
084      generate(source, "Messages.properties", "validator-messages-en.po",    null, 2);
085      for (String name : ini.getProperties("languages").keySet()) {
086        generate(source, "rendering-phrases.properties",  "rendering-phrases-"+name+".po",    "rendering-phrases_"+name+".properties", ini.getIntegerProperty("languages", name));
087        generate(source, "Messages.properties", "validator-messages-"+name+".po",    "Messages_"+name+".properties", ini.getIntegerProperty("languages", name));        
088      }
089
090      System.out.println("Finished");
091    } 
092  }
093
094  private boolean checkState(String source, String core, String igpub, String pascal) throws IOException {
095    System.out.println("Checking...");
096    List<PropertyValue> props = loadProperties(Utilities.path(source, "rendering-phrases.properties"), true);
097    List<ConstantDefinition> consts = loadConstants(Utilities.path(core, "/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/i18n/RenderingI18nContext.java"));
098    boolean ok = true;
099    for (ConstantDefinition cd : consts) {
100      boolean found = false;
101      for (PropertyValue p : props) {
102        String pn = p.getBaseName();        
103        if (pn.equals(cd.sname) || p.getName().equals(cd.sname)) {
104          found = true;
105          p.used = true;
106        }
107      }
108      cd.defined = found;
109    }
110    scanJavaSource(core, consts, "RenderingI18nContext", "RenderingContext");
111    scanJavaSource(igpub, consts, "RenderingI18nContext", "RenderingContext");
112    scanPascalSource(pascal, props);
113    
114    Set<String> pns = new HashSet<>();
115    for (PropertyValue p : props) {
116      if (!p.used) {
117        ok = false;
118        System.out.println("Error: PV "+p.getName()+ " provided but not used");   
119      }
120      if (!pns.contains(p.getName())) {
121        pns.add(p.getName());
122      } else {
123        System.out.println("Error: PV "+p.getName()+ " duplicated");
124      }
125      if (p.getValue().contains("\\n")) {
126        System.out.println("Error: PV "+p.getName()+ " has a \\n");
127      }
128    }
129    
130    for (ConstantDefinition cd : consts) {
131      if (!cd.defined && !cd.used) {
132        System.out.println("Error: "+cd.name+ " not defined or used");        
133        ok = false;
134      } else if (!cd.defined) {
135        ok = false;
136        System.out.println("Error: msg for "+cd.name+ " not found at "+cd.sname);
137      } else if (!cd.used) {
138        System.out.println("Warning: const "+cd.name+ " not used");
139        // ok = false;
140      }
141    }
142
143    props = loadProperties(Utilities.path(source, "Messages.properties"), true);
144    consts = loadConstants(Utilities.path(core, "/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/i18n/I18nConstants.java"));
145    for (ConstantDefinition cd : consts) {
146      boolean found = false;
147      for (PropertyValue p : props) {
148        String pn = p.getBaseName();
149        if (pn.equals(cd.sname) || p.getName().equals(cd.sname)) {
150          found = true;
151          p.used = true;
152        }
153      }
154      cd.defined = found;
155    }
156
157    scanJavaSource(core, consts, "I18nConstants");
158    scanJavaSource(igpub, consts, "I18nConstants");
159    scanPascalSource(pascal, props);
160
161    pns = new HashSet<>();
162    for (PropertyValue p : props) {
163      if (!p.used) {
164        ok = false;
165        System.out.println("Error: PV "+p.getName()+ " provided but not used");   
166      }
167      if (!pns.contains(p.getName())) {
168        pns.add(p.getName());
169      } else {
170        System.out.println("Error: PV "+p.getName()+ " duplicated");
171      }      
172      if (p.getValue().contains("\\n")) {
173        System.out.println("Error: PV "+p.getName()+ " has a \\n");
174      }
175
176    }
177    
178    for (ConstantDefinition cd : consts) {
179      if (!cd.defined && !cd.used) {
180        System.out.println("Error: "+cd.name+ " not defined or used");        
181        ok = false;
182      } else if (!cd.defined) {
183        ok = false;
184        System.out.println("Error: msg for "+cd.name+ " not found @ "+cd.sname);
185      } else if (!cd.used) {
186        System.out.println("Warning: const "+cd.name+ " not used");
187        // ok = false;
188      }
189    }
190    if (ok) {
191      System.out.println("No Errors Found");
192    } else {
193      System.out.println("Errors Found, so not continuing");  
194    }
195
196    return ok;
197  }
198
199  private class JavaScanner implements IDirectoryVisitorImplementation {
200    List<ConstantDefinition> consts;
201    List<String> names;
202    
203    @Override
204    public boolean enterDirectory(File f) throws IOException {
205      return !Utilities.existsInList(f.getName(), "model", "formats");
206    }
207
208    @Override
209    public boolean visitFile(File file) throws IOException {
210      String source = FileUtilities.fileToString(file);
211      for (ConstantDefinition cd : consts) {
212        if (!cd.used) {
213          boolean found = false;
214          for (String n : names) {
215            if (source.contains(n+"."+cd.name+",")) {
216              found = true;
217            } 
218            if (source.contains(n+"."+cd.name+")")) {
219              found = true;
220            } 
221            if (source.contains(n+"."+cd.name+" :")) {
222              found = true;
223            } 
224            if (source.contains(n+"."+cd.name+";")) {
225              found = true;
226            } 
227          } 
228          if (found) {
229            cd.used = true;
230          }
231        }  
232      }
233      return true;
234    }
235  }
236  
237  private void scanJavaSource(String path, List<ConstantDefinition> consts, String... names) throws FileNotFoundException, IOException {
238    JavaScanner scanner = new JavaScanner();
239    scanner.consts = consts;
240    scanner.names = new ArrayList<String>();
241    for (String s : names) {
242      scanner.names.add(s);
243    }
244    DirectoryVisitor.visitDirectory(scanner, path, "java");
245  }
246
247  private class PascalScanner implements IDirectoryVisitorImplementation {
248    private List<PropertyValue> defs;
249    
250    @Override
251    public boolean enterDirectory(File directory) throws IOException {
252      return true;
253    }
254
255    @Override
256    public boolean visitFile(File file) throws IOException {
257      String source = FileUtilities.fileToString(file);
258      for (PropertyValue pv : defs) {
259        if (!pv.used) {
260          boolean found = false;
261          String pn = pv.getBaseName();
262          if (source.contains("'"+pn+"'")) {
263            found = true;
264          } 
265          if (found) {
266            pv.used = true;
267          }
268        }
269      }
270      return true;
271    }
272  }
273  
274  private void scanPascalSource(String path, List<PropertyValue> defs) throws FileNotFoundException, IOException {
275    PascalScanner scanner = new PascalScanner();
276    scanner.defs = defs;
277    DirectoryVisitor.visitDirectory(scanner, path, "pas");
278  }
279
280  
281  private List<ConstantDefinition> loadConstants(String path) throws FileNotFoundException, IOException {
282    List<ConstantDefinition> res = new ArrayList<POGenerator.ConstantDefinition>();
283    for (String line : FileUtilities.fileToLines(path)) {
284      if (line.contains("public static final String") && !line.trim().startsWith("//")) {
285        int i = line.indexOf("public static final String") + "public static final String".length();
286        String[] p = line.substring(i).split("\\=");
287        if (p.length == 2) {
288          String n = p[0].trim();
289          String v = p[1].trim().replace("\"", "").replace(";", "");
290          ConstantDefinition cd = new ConstantDefinition();
291          cd.name = n;
292          cd.sname = v;
293          res.add(cd);
294        }
295      }
296    }
297    return res;
298  }
299
300  enum PluralMode {
301    NONE, ONE, OTHER
302  }
303
304  private void generate(String sourceDirectory, String propertiesFileName, String poFileName, String targetPropertiesFileName, int count) throws IOException {
305    // load the destination file 
306    // load the source file 
307    // update the destination object set for changes from the source file
308    // save the destination file 
309    String poFilePath = Utilities.path(sourceDirectory, "source", poFileName);
310    POSource poObjects;
311    if (ManagedFileAccess.file(poFilePath).exists()) {
312      poObjects = POSource.loadPOFile(poFilePath);
313    } else {
314      poObjects = new POSource();
315    }
316    List<PropertyValue> javaProperties = loadProperties(Utilities.path(sourceDirectory, propertiesFileName), false);
317    for (PropertyValue javaProperty : javaProperties) {
318      String name = javaProperty.getName();
319      PluralMode pluralMode = PluralMode.NONE;
320      if (name.endsWith("_one")) {
321        pluralMode = PluralMode.ONE;
322        name = name.substring(0, name.length() - 4);
323      } else if (name.endsWith("_other")) {
324        pluralMode = PluralMode.OTHER;
325        name = name.substring(0, name.length() - 6);
326      } 
327
328      POObject poObject = findObject(poObjects.getPOObjects(), name);
329      if (poObject == null) {
330        if (pluralMode == PluralMode.OTHER) {
331          throw new Error("Not right");
332        }
333        poObject = new POObject();
334        poObject.setId(name);
335        poObject.setComment(name);
336        poObjects.getPOObjects().add(poObject);
337        poObject.setMsgid(javaProperty.getValue());
338        poObject.setOrphan(false);
339      } else {
340        updatePOObject(poObject, pluralMode, javaProperty.getValue());
341      }
342    }
343    poObjects.getPOObjects().removeIf(o -> o.isOrphan());
344    Collections.sort(poObjects.getPOObjects(), new POObjectSorter());
345    markDuplicatePOObjects(poObjects);
346    noTrans = poObjects.savePOFile(Utilities.path(sourceDirectory, "source", poFileName), count, noTrans);
347    if (targetPropertiesFileName != null) {
348      savePropertiesFile(Utilities.path(sourceDirectory, targetPropertiesFileName), poObjects.getPOObjects());
349    }
350  }
351
352  private static void markDuplicatePOObjects(POSource poObjects) {
353    Map<String, Integer> sources = new HashMap<>();
354
355    for (POObject poObject : poObjects.getPOObjects()) {
356      if (sources.containsKey(poObject.getMsgid())) {
357        Integer count = sources.get(poObject.getMsgid())+1;
358        sources.put(poObject.getMsgid(), count);
359//        System.out.println("Duplicate in "+dest.substring(dest.lastIndexOf("/")+1)+": "+o.msgid+" on ("+CommaSeparatedStringBuilder.join(",", listIds(objects, o.msgid))+")");
360      } else {
361        sources.put(poObject.getMsgid(), 1);
362      }
363    }
364    for (POObject poObject : poObjects.getPOObjects()) {
365      Integer count = sources.get(poObject.getMsgid());
366      if (count > 1) {
367        poObject.setDuplicate(true);
368      }
369    }
370  }
371
372  private void updatePOObject(POObject poObject, PluralMode mode, String value) {
373    poObject.setOrphan(false);
374    if (poObject.getComment() == null) {
375      poObject.setComment(poObject.getId());
376    }
377    if (mode == PluralMode.NONE) {
378      if (!value.equals(poObject.getMsgid())) {
379        // the english string has changed, and the other language string is now out of date
380        if (poObject.getOldMsgId() != null && !poObject.getMsgstr().isEmpty()) {
381          poObject.setOldMsgId(poObject.getMsgid());
382        }
383        poObject.setMsgid(value);
384        for (int i = 0; i < poObject.getMsgstr().size(); i++) {
385          if (!Utilities.noString(poObject.getMsgstr().get(i))) {
386            poObject.getMsgstr().set(i, "!!"+poObject.getMsgstr().get(i));
387          }
388        }
389      } else {
390        poObject.setOldMsgId(null);
391      }
392    } else if (mode == PluralMode.ONE) {
393      if (!value.equals(poObject.getMsgid())) {
394        // the english string has changed, and the other language string is now out of date 
395        if (poObject.getOldMsgId() != null && !poObject.getMsgstr().isEmpty()) {
396          poObject.setOldMsgId(poObject.getMsgid());
397        }
398        poObject.setMsgid(value);
399        if (poObject.getMsgstr().size() > 0 && !Utilities.noString(poObject.getMsgstr().get(0))) {
400          poObject.getMsgstr().set(0, "!!"+poObject.getMsgstr().get(0));
401        }
402      } else {
403        poObject.setOldMsgId(null);
404      }
405    } else if (mode == PluralMode.OTHER) {
406      if (!value.equals(poObject.getMsgidPlural())) {
407        // the english string has changed, and the other language string is now out of date 
408//        if (o.oldMsgId != null) {
409//          o.oldMsgId = o.msgid;
410//        }
411        poObject.setMsgidPlural(value);
412        if (poObject.getMsgstr().size() > 1 && !Utilities.noString(poObject.getMsgstr().get(1))) {
413          poObject.getMsgstr().set(1, "!!"+poObject.getMsgstr().get(1));
414        }
415      } else {
416        poObject.setOldMsgId(null);
417      }
418    }
419  }
420
421  private POObject findObject(List<POObject> poObjects, String name) {
422    for (POObject poObject : poObjects) {
423      if (poObject.getId() != null && poObject.getId().equals(name)) {
424        return poObject;
425      }
426    }
427    return null;
428  }
429
430  private List<PropertyValue> loadProperties(String source, boolean checking) throws IOException {
431    List<PropertyValue> res = new ArrayList<>();
432    File src = ManagedFileAccess.file(source);
433    BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(src), StandardCharsets.UTF_8));
434    for (String line = reader.readLine(); line != null; line = reader.readLine()) {
435      if (!line.startsWith("#") && line.contains("=")) {
436        String n = line.substring(0, line.indexOf("=")).trim();
437        String v = line.substring(line.indexOf("=")+1).trim();
438        if (checking || !(v.length() == 3 && v.startsWith("{") && v.endsWith("}"))) {
439          res.add(new PropertyValue(n, v));
440        }
441      } 
442    }
443    return res;
444  }
445
446  private void savePropertiesFile(String tgt, List<POObject> objects) throws IOException {
447    String nameLine = FileUtilities.fileToLines(tgt).get(0);
448    String[] parts = nameLine.substring(1).trim().split("\\=");
449    String[] names = parts[1].split("\\,");
450    
451    StringBuilder b = new StringBuilder();
452    b.append(nameLine+"\r\n");
453    for (POObject o : objects) {
454      if (o.getMsgidPlural() == null) {
455        String v= o.getMsgstr().size() > 0 ? o.getMsgstr().get(0) : "";
456        if (!Utilities.noString(v)) {
457          if (v.startsWith("!!")) {
458            v = v.substring(2);
459          }
460          b.append(o.getId()+" = "+v+"\r\n");
461        }
462      } else {
463        for (int i = 0; i < names.length; i++) {
464          String v = (o.getMsgstr().size() > i ? o.getMsgstr().get(i) : "");
465          if (!Utilities.noString(v)) {
466            if (v.startsWith("!!")) {
467              v = v.substring(2);
468            }
469            b.append(o.getId()+"_"+names[i].trim()+" = "+v+"\r\n");
470          }
471        }
472      }
473    }
474    
475    FileUtilities.stringToFile(b.toString(), tgt);
476  }
477
478  
479}