001package org.hl7.fhir.utilities.cache;
002
003/*
004  Copyright (c) 2011+, HL7, Inc.
005  All rights reserved.
006  
007  Redistribution and use in source and binary forms, with or without modification, 
008  are permitted provided that the following conditions are met:
009    
010   * Redistributions of source code must retain the above copyright notice, this 
011     list of conditions and the following disclaimer.
012   * Redistributions in binary form must reproduce the above copyright notice, 
013     this list of conditions and the following disclaimer in the documentation 
014     and/or other materials provided with the distribution.
015   * Neither the name of HL7 nor the names of its contributors may be used to 
016     endorse or promote products derived from this software without specific 
017     prior written permission.
018  
019  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
020  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
021  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 
022  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 
023  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 
024  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
025  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
026  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
027  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
028  POSSIBILITY OF SUCH DAMAGE.
029  
030 */
031
032
033import com.google.gson.GsonBuilder;
034import com.google.gson.JsonArray;
035import com.google.gson.JsonElement;
036import com.google.gson.JsonObject;
037import org.apache.commons.io.FileUtils;
038import org.hl7.fhir.exceptions.FHIRException;
039import org.hl7.fhir.utilities.IniFile;
040import org.hl7.fhir.utilities.TextFile;
041import org.hl7.fhir.utilities.Utilities;
042import org.hl7.fhir.utilities.cache.NpmPackage.NpmPackageFolder;
043import org.hl7.fhir.utilities.json.JSONUtil;
044import org.slf4j.Logger;
045import org.slf4j.LoggerFactory;
046
047import java.io.File;
048import java.io.FileInputStream;
049import java.io.FileNotFoundException;
050import java.io.IOException;
051import java.io.InputStream;
052import java.io.RandomAccessFile;
053import java.net.HttpURLConnection;
054import java.net.URL;
055import java.net.URLConnection;
056import java.nio.channels.FileChannel;
057import java.nio.channels.FileLock;
058import java.sql.Timestamp;
059import java.text.ParseException;
060import java.text.SimpleDateFormat;
061import java.time.Instant;
062import java.util.ArrayList;
063import java.util.Collections;
064import java.util.Comparator;
065import java.util.Date;
066import java.util.HashMap;
067import java.util.List;
068import java.util.Locale;
069import java.util.Map;
070import java.util.Map.Entry;
071
072/**
073 * This is a package cache manager implementation that uses a local disk cache
074 *
075 * <p>
076 * API:
077 * <p>
078 * constructor
079 * getPackageUrl
080 * getPackageId
081 * findPackageCache
082 * addPackageToCache
083 *
084 * @author Grahame Grieve
085 */
086public class FilesystemPackageCacheManager extends BasePackageCacheManager implements IPackageCacheManager {
087
088  public static final String PRIMARY_SERVER = "http://packages.fhir.org";
089  public static final String SECONDARY_SERVER = "http://packages2.fhir.org/packages";
090  //  private static final String SECONDARY_SERVER = "http://local.fhir.org:960/packages";
091  public static final String PACKAGE_REGEX = "^[a-z][a-z0-9\\_\\-]*(\\.[a-z0-9\\_\\-]+)+$";
092  public static final String PACKAGE_VERSION_REGEX = "^[a-z][a-z0-9\\_\\-]*(\\.[a-z0-9\\_\\-]+)+\\#[a-z0-9\\-\\_]+(\\.[a-z0-9\\-\\_]+)*$";
093  private static final Logger ourLog = LoggerFactory.getLogger(FilesystemPackageCacheManager.class);
094  private static final String CACHE_VERSION = "3"; // second version - see wiki page
095  private String cacheFolder;
096  private boolean progress = true;
097  private List<NpmPackage> temporaryPackages = new ArrayList<>();
098  private boolean buildLoaded = false;
099  private Map<String, String> ciList = new HashMap<String, String>();
100  private JsonArray buildInfo;
101
102  /**
103   * Constructor
104   */
105  public FilesystemPackageCacheManager(boolean userMode, int toolsVersion) throws IOException {
106    addPackageServer(PRIMARY_SERVER);
107    addPackageServer(SECONDARY_SERVER);
108
109    if (userMode)
110      cacheFolder = Utilities.path(System.getProperty("user.home"), ".fhir", "packages");
111    else
112      cacheFolder = Utilities.path("var", "lib", ".fhir", "packages");
113    if (!(new File(cacheFolder).exists()))
114      Utilities.createDirectory(cacheFolder);
115    if (!(new File(Utilities.path(cacheFolder, "packages.ini")).exists()))
116      TextFile.stringToFile("[cache]\r\nversion=" + CACHE_VERSION + "\r\n\r\n[urls]\r\n\r\n[local]\r\n\r\n", Utilities.path(cacheFolder, "packages.ini"), false);
117    createIniFile();
118  }
119
120  public void loadFromFolder(String packagesFolder) throws IOException {
121    File[] files = new File(packagesFolder).listFiles();
122    if (files != null) {
123      for (File f : files) {
124        if (f.getName().endsWith(".tgz")) {
125          temporaryPackages.add(NpmPackage.fromPackage(new FileInputStream(f)));
126        }
127      }
128    }
129  }
130
131  public String getFolder() {
132    return cacheFolder;
133  }
134
135  private List<String> sorted(String[] keys) {
136    List<String> names = new ArrayList<String>();
137    for (String s : keys)
138      names.add(s);
139    Collections.sort(names);
140    return names;
141  }
142
143  private NpmPackage loadPackageInfo(String path) throws IOException {
144    NpmPackage pi = NpmPackage.fromFolder(path);
145    return pi;
146  }
147
148  private JsonObject fetchJson(String source) throws IOException {
149    URL url = new URL(source);
150    URLConnection c = url.openConnection();
151    return (JsonObject) new com.google.gson.JsonParser().parse(TextFile.streamToString(c.getInputStream()));
152  }
153
154  private void clearCache() throws IOException {
155    for (File f : new File(cacheFolder).listFiles()) {
156      if (f.isDirectory()) {
157        new CacheLock(f.getName()).doWithLock(() -> {
158          Utilities.clearDirectory(f.getAbsolutePath());
159          try {
160            FileUtils.deleteDirectory(f);
161          } catch (Exception e1) {
162            try {
163              FileUtils.deleteDirectory(f);
164            } catch (Exception e2) {
165              // just give up
166            }
167          }
168          return null; // must return something
169        });
170      } else if (!f.getName().equals("packages.ini"))
171        FileUtils.forceDelete(f);
172    }
173    IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
174    ini.removeSection("packages");
175    ini.save();
176  }
177
178  private void createIniFile() throws IOException {
179    IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
180    boolean save = false;
181    String v = ini.getStringProperty("cache", "version");
182    if (!CACHE_VERSION.equals(v)) {
183      clearCache();
184      ini.setStringProperty("cache", "version", CACHE_VERSION, null);
185      ini.save();
186    }
187  }
188
189  private void checkValidVersionString(String version, String id) {
190    if (Utilities.noString(version)) {
191      throw new FHIRException("Cannot add package " + id + " to the package cache - a version must be provided");
192    }
193    if (version.startsWith("file:")) {
194      throw new FHIRException("Cannot add package " + id + " to the package cache - the version '" + version + "' is illegal in this context");
195    }
196    for (char ch : version.toCharArray()) {
197      if (!Character.isAlphabetic(ch) && !Character.isDigit(ch) && !Utilities.existsInList(ch, '.', '-')) {
198        throw new FHIRException("Cannot add package " + id + " to the package cache - the version '" + version + "' is illegal (ch '" + ch + "'");
199      }
200    }
201  }
202
203  private void listSpecs(Map<String, String> specList, String server) throws IOException {
204    CachingPackageClient pc = new CachingPackageClient(server);
205    List<PackageClient.PackageInfo> matches = pc.search(null, null, null, false);
206    for (PackageClient.PackageInfo m : matches) {
207      if (!specList.containsKey(m.getId())) {
208        specList.put(m.getId(), m.getUrl());
209      }
210    }
211  }
212
213  protected InputStreamWithSrc loadFromPackageServer(String id, String version) {
214    InputStreamWithSrc retVal = super.loadFromPackageServer(id, version);
215    if (retVal != null) {
216      return retVal;
217    }
218
219    // ok, well, we'll try the old way
220    return fetchTheOldWay(id, version);
221  }
222
223  public String getLatestVersion(String id) throws IOException {
224    for (String nextPackageServer : getPackageServers()) {
225      CachingPackageClient pc = new CachingPackageClient(nextPackageServer);
226      try {
227        return pc.getLatestVersion(id);
228      } catch (IOException e) {
229        ourLog.info("Failed to determine latest version of package {} from server: {}", id, nextPackageServer);
230      }
231    }
232
233    return fetchVersionTheOldWay(id);
234  }
235
236  private NpmPackage loadPackageFromFile(String id, String folder) throws IOException {
237    File f = new File(Utilities.path(folder, id));
238    if (!f.exists()) {
239      throw new FHIRException("Package '" + id + "  not found in folder " + folder);
240    }
241    if (!f.isDirectory()) {
242      throw new FHIRException("File for '" + id + "  found in folder " + folder + ", not a folder");
243    }
244    File fp = new File(Utilities.path(folder, id, "package", "package.json"));
245    if (!fp.exists()) {
246      throw new FHIRException("Package '" + id + "  found in folder " + folder + ", but does not contain a package.json file in /package");
247    }
248    return NpmPackage.fromFolder(f.getAbsolutePath());
249  }
250
251  /**
252   * Clear the cache
253   *
254   * @throws IOException
255   */
256  public void clear() throws IOException {
257    clearCache();
258  }
259
260  // ========================= Utilities ============================================================================
261
262  /**
263   * Remove a particular package from the cache
264   *
265   * @param id
266   * @param ver
267   * @throws IOException
268   */
269  public void removePackage(String id, String ver) throws IOException {
270    new CacheLock(id + "#" + ver).doWithLock(() -> {
271      String f = Utilities.path(cacheFolder, id + "#" + ver);
272      File ff = new File(f);
273      if (ff.exists()) {
274        Utilities.clearDirectory(f);
275        IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
276        ini.removeProperty("packages", id + "#" + ver);
277        ini.save();
278        ff.delete();
279      }
280      return null;
281    });
282  }
283
284  /**
285   * Load the identified package from the cache - it it exists
286   * <p>
287   * This is for special purpose only (testing, control over speed of loading).
288   * Generally, use the loadPackage method
289   *
290   * @param id
291   * @param version
292   * @return
293   * @throws IOException
294   */
295  @Override
296  public NpmPackage loadPackageFromCacheOnly(String id, String version) throws IOException {
297    if (!Utilities.noString(version) && version.startsWith("file:")) {
298      return loadPackageFromFile(id, version.substring(5));
299    }
300
301    for (NpmPackage p : temporaryPackages) {
302      if (p.name().equals(id) && ("current".equals(version) || "dev".equals(version) || p.version().equals(version))) {
303        return p;
304      }
305      if (p.name().equals(id) && Utilities.noString(version)) {
306        return p;
307      }
308    }
309    for (String f : sorted(new File(cacheFolder).list())) {
310      if (f.equals(id + "#" + version) || (Utilities.noString(version) && f.startsWith(id + "#"))) {
311        return loadPackageInfo(Utilities.path(cacheFolder, f));
312      }
313    }
314    if ("dev".equals(version))
315      return loadPackageFromCacheOnly(id, "current");
316    else
317      return null;
318  }
319
320  /**
321   * Add an already fetched package to the cache
322   */
323  @Override
324  public NpmPackage addPackageToCache(String id, String version, InputStream packageTgzInputStream, String sourceDesc) throws IOException {
325    checkValidVersionString(version, id);
326    if (progress) {
327      System.out.println("Installing " + id + "#" + (version == null ? "?" : version) + " to the package cache");
328      System.out.print("  Fetching:");
329    }
330
331    NpmPackage npm = NpmPackage.fromPackage(packageTgzInputStream, sourceDesc, true);
332
333    if (progress) {
334      System.out.println();
335      System.out.print("  Installing: ");
336    }
337    if (npm.name() == null || id == null || !id.equals(npm.name())) {
338      if (!id.equals("hl7.fhir.r5.core")) {// temporary work around
339        throw new IOException("Attempt to import a mis-identified package. Expected " + id + ", got " + npm.name());
340      }
341    }
342    if (version == null)
343      version = npm.version();
344
345    String v = version;
346    return new CacheLock(id + "#" + version).doWithLock(() -> {
347      NpmPackage pck = null;
348      String packRoot = Utilities.path(cacheFolder, id + "#" + v);
349      try {
350        // ok, now we have a lock on it... check if something created it while we were waiting
351        if (!new File(packRoot).exists() || Utilities.existsInList(v, "current", "dev")) {
352          Utilities.createDirectory(packRoot);
353          try {
354            Utilities.clearDirectory(packRoot);
355          } catch (Throwable t) {
356            System.out.println("Unable to clear directory: "+packRoot+": "+t.getMessage()+" - this may cause problems later");
357          }
358
359          int i = 0;
360          int c = 0;
361          int size = 0;
362          for (Entry<String, NpmPackageFolder> e : npm.getFolders().entrySet()) {
363            String dir = e.getKey().equals("package") ? Utilities.path(packRoot, "package") : Utilities.path(packRoot, "package", e.getKey());
364            if (!(new File(dir).exists()))
365              Utilities.createDirectory(dir);
366            for (Entry<String, byte[]> fe : e.getValue().getContent().entrySet()) {
367              String fn = Utilities.path(dir, fe.getKey());
368              byte[] cnt = fe.getValue();
369              TextFile.bytesToFile(cnt, fn);
370              size = size + cnt.length;
371              i++;
372              if (progress && i % 50 == 0) {
373                c++;
374                System.out.print(".");
375                if (c == 120) {
376                  System.out.println("");
377                  System.out.print("  ");
378                  c = 2;
379                }
380              }
381            }
382          }
383
384
385          IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
386          ini.setTimeStampFormat("yyyyMMddhhmmss");
387          ini.setTimestampProperty("packages", id + "#" + v, Timestamp.from(Instant.now()), null);
388          ini.setIntegerProperty("package-sizes", id + "#" + v, size, null);
389          ini.save();
390          if (progress)
391            System.out.println(" done.");
392        }
393        pck = loadPackageInfo(packRoot);
394        if (!id.equals(JSONUtil.str(npm.getNpm(), "name")) || !v.equals(JSONUtil.str(npm.getNpm(), "version"))) {
395          if (!id.equals(JSONUtil.str(npm.getNpm(), "name"))) {
396            npm.getNpm().addProperty("original-name", JSONUtil.str(npm.getNpm(), "name"));
397            npm.getNpm().remove("name");
398            npm.getNpm().addProperty("name", id);
399          }
400          if (!v.equals(JSONUtil.str(npm.getNpm(), "version"))) {
401            npm.getNpm().addProperty("original-version", JSONUtil.str(npm.getNpm(), "version"));
402            npm.getNpm().remove("version");
403            npm.getNpm().addProperty("version", v);
404          }
405          TextFile.stringToFile(new GsonBuilder().setPrettyPrinting().create().toJson(npm.getNpm()), Utilities.path(cacheFolder, id + "#" + v, "package", "package.json"), false);
406        }
407      } catch (Exception e) {
408        try {
409          // don't leave a half extracted package behind
410          System.out.println("Clean up package " + packRoot + " because installation failed: " + e.getMessage());
411          e.printStackTrace();
412          Utilities.clearDirectory(packRoot);
413          new File(packRoot).delete();
414        } catch (Exception ei) {
415          // nothing
416        }
417        throw e;
418      }
419      return pck;
420    });
421  }
422
423  @Override
424  public String getPackageUrl(String packageId) throws IOException {
425    String result = super.getPackageUrl(packageId);
426    if (result == null) {
427      result = getPackageUrlFromBuildList(packageId);
428    }
429
430    return result;
431  }
432
433  public void listAllIds(Map<String, String> specList) throws IOException {
434    for (NpmPackage p : temporaryPackages) {
435      specList.put(p.name(), p.canonical());
436    }
437    for (String next : getPackageServers()) {
438      listSpecs(specList, next);
439    }
440    addCIBuildSpecs(specList);
441  }
442
443  @Override
444  public NpmPackage loadPackage(String id, String version) throws FHIRException, IOException {
445    //ok, try to resolve locally
446    if (!Utilities.noString(version) && version.startsWith("file:")) {
447      return loadPackageFromFile(id, version.substring(5));
448    }
449
450    if (version == null && id.contains("#")) {
451      version = id.substring(id.indexOf("#")+1);
452      id = id.substring(0, id.indexOf("#"));
453    }
454    NpmPackage p = loadPackageFromCacheOnly(id, version);
455    if (p != null) {
456      if ("current".equals(version)) {
457        p = checkCurrency(id, p);
458      }
459      if (p != null)
460        return p;
461    }
462
463    if ("dev".equals(version)) {
464      p = loadPackageFromCacheOnly(id, "current");
465      p = checkCurrency(id, p);
466      if (p != null)
467        return p;
468      version = "current";
469    }
470
471    // nup, don't have it locally (or it's expired)
472    FilesystemPackageCacheManager.InputStreamWithSrc source;
473    if ("current".equals(version)) {
474      // special case - fetch from ci-build server
475      source = loadFromCIBuild(id);
476    } else {
477      source = loadFromPackageServer(id, version);
478    }
479    if (source == null) {
480      throw new FHIRException("Unable to find package "+id+"#"+version);
481    }
482    return addPackageToCache(id, version == null ? source.version : version, source.stream, source.url);
483  }
484
485  private InputStream fetchFromUrlSpecific(String source, boolean optional) throws FHIRException {
486    try {
487      URL url = new URL(source);
488      URLConnection c = url.openConnection();
489      return c.getInputStream();
490    } catch (Exception e) {
491      if (optional)
492        return null;
493      else
494        throw new FHIRException(e.getMessage(), e);
495    }
496  }
497
498  private InputStreamWithSrc loadFromCIBuild(String id) throws IOException {
499    checkBuildLoaded();
500    if (ciList.containsKey(id)) {
501      InputStream stream = fetchFromUrlSpecific(Utilities.pathURL(ciList.get(id), "package.tgz"), false);
502      return new InputStreamWithSrc(stream, Utilities.pathURL(ciList.get(id), "package.tgz"), "current");
503    } else if (id.startsWith("hl7.fhir.r5")) {
504      InputStream stream = fetchFromUrlSpecific(Utilities.pathURL("http://build.fhir.org", id + ".tgz"), false);
505      return new InputStreamWithSrc(stream, Utilities.pathURL("http://build.fhir.org", id + ".tgz"), "current");
506    } else {
507      throw new FHIRException("The package '" + id + "' has not entry on the current build server");
508    }
509  }
510
511  private String getPackageUrlFromBuildList(String packageId) throws IOException {
512    checkBuildLoaded();
513    for (JsonElement n : buildInfo) {
514      JsonObject o = (JsonObject) n;
515      if (packageId.equals(JSONUtil.str(o, "package-id"))) {
516        return JSONUtil.str(o, "url");
517      }
518    }
519    return null;
520  }
521
522  private void addCIBuildSpecs(Map<String, String> specList) throws IOException {
523    checkBuildLoaded();
524    for (JsonElement n : buildInfo) {
525      JsonObject o = (JsonObject) n;
526      if (!specList.containsKey(JSONUtil.str(o, "package-id"))) {
527        specList.put(JSONUtil.str(o, "package-id"), JSONUtil.str(o, "url"));
528      }
529    }
530  }
531
532  @Override
533  public String getPackageId(String canonicalUrl) throws IOException {
534    String retVal = super.getPackageId(canonicalUrl);
535
536    if (retVal == null) {
537      retVal = getPackageIdFromBuildList(canonicalUrl);
538    }
539
540    return retVal;
541  }
542
543
544  // ========================= Package Mgmt API =======================================================================
545
546  private String getPackageIdFromBuildList(String canonical) throws IOException {
547    if (canonical == null) {
548      return null;
549    }
550    checkBuildLoaded();
551    if (buildInfo != null) {
552      for (JsonElement n : buildInfo) {
553        JsonObject o = (JsonObject) n;
554        if (canonical.equals(JSONUtil.str(o, "url"))) {
555          return JSONUtil.str(o, "package-id");
556        }
557      }
558      for (JsonElement n : buildInfo) {
559        JsonObject o = (JsonObject) n;
560        if (JSONUtil.str(o, "url").startsWith(canonical + "/ImplementationGuide/")) {
561          return JSONUtil.str(o, "package-id");
562        }
563      }
564    }
565    return null;
566  }
567
568  private NpmPackage checkCurrency(String id, NpmPackage p) throws IOException {
569    checkBuildLoaded();
570    // special case: current versions roll over, and we have to check their currency
571    try {
572      String url = ciList.get(id);
573      JsonObject json = fetchJson(Utilities.pathURL(url, "package.manifest.json"));
574      String currDate = JSONUtil.str(json, "date");
575      String packDate = p.date();
576      if (!currDate.equals(packDate))
577        return null; // nup, we need a new copy
578      return p;
579    } catch (Exception e) {
580      return p;
581    }
582  }
583
584  private boolean checkBuildLoaded() {
585    if (buildLoaded)
586      return true;
587    try {
588      loadFromBuildServer();
589    } catch (Exception e) {
590      System.out.println("Error connecting to build server - running without build (" + e.getMessage() + ")");
591    }
592    return false;
593  }
594
595  private void loadFromBuildServer() throws IOException {
596    URL url = new URL("https://build.fhir.org/ig/qas.json?nocache=" + System.currentTimeMillis());
597    HttpURLConnection connection = (HttpURLConnection) url.openConnection();
598    connection.setRequestMethod("GET");
599    InputStream json = connection.getInputStream();
600    buildInfo = (JsonArray) new com.google.gson.JsonParser().parse(TextFile.streamToString(json));
601
602    List<BuildRecord> builds = new ArrayList<>();
603
604    for (JsonElement n : buildInfo) {
605      JsonObject o = (JsonObject) n;
606      if (o.has("url") && o.has("package-id") && o.get("package-id").getAsString().contains(".")) {
607        String u = o.get("url").getAsString();
608        if (u.contains("/ImplementationGuide/"))
609          u = u.substring(0, u.indexOf("/ImplementationGuide/"));
610        builds.add(new BuildRecord(u, o.get("package-id").getAsString(), getRepo(o.get("repo").getAsString()), readDate(o.get("date").getAsString())));
611      }
612    }
613    Collections.sort(builds, new BuildRecordSorter());
614    for (BuildRecord bld : builds) {
615      if (!ciList.containsKey(bld.getPackageId())) {
616        ciList.put(bld.getPackageId(), "https://build.fhir.org/ig/" + bld.getRepo());
617      }
618    }
619    buildLoaded = true; // whether it succeeds or not
620  }
621
622  //  private String buildPath(String url) {
623//    for (JsonElement e : buildInfo) {
624//      JsonObject j = (JsonObject) e;
625//      if (j.has("url") && (url.equals(j.get("url").getAsString()) || j.get("url").getAsString().startsWith(url+"/ImplementationGuide"))) {
626//        return "https://build.fhir.org/ig/"+j.get("repo").getAsString();
627//      }
628//    }
629//    return null;
630//  }
631//
632  private String getRepo(String path) {
633    String[] p = path.split("\\/");
634    return p[0] + "/" + p[1];
635  }
636
637  private Date readDate(String s) {
638    SimpleDateFormat sdf = new SimpleDateFormat("EEE, dd MMM, yyyy HH:mm:ss Z", new Locale("en", "US"));
639    try {
640      return sdf.parse(s);
641    } catch (ParseException e) {
642      e.printStackTrace();
643      return new Date();
644    }
645  }
646
647  // ----- the old way, from before package server, while everything gets onto the package server
648  private InputStreamWithSrc fetchTheOldWay(String id, String v) {
649    String url = getUrlForPackage(id);
650    if (url == null) {
651      try {
652        url = getPackageUrlFromBuildList(id);
653      } catch (Exception e) {
654        url = null;
655      }
656    }
657    if (url == null) {
658      throw new FHIRException("Unable to resolve package id " + id + "#" + v);
659    }
660    if (url.contains("/ImplementationGuide/")) {
661      url = url.substring(0, url.indexOf("/ImplementationGuide/"));
662    }
663    String pu = Utilities.pathURL(url, "package-list.json");
664    String aurl = pu;
665    JsonObject json;
666    try {
667      json = fetchJson(pu);
668    } catch (Exception e) {
669      String pv = Utilities.pathURL(url, v, "package.tgz");
670      try {
671        aurl = pv;
672        InputStreamWithSrc src = new InputStreamWithSrc(fetchFromUrlSpecific(pv, true), pv, v);
673        return src;
674      } catch (Exception e1) {
675        throw new FHIRException("Error fetching package directly (" + pv + "), or fetching package list for " + id + " from " + pu + ": " + e1.getMessage(), e1);
676      }
677    }
678    if (!id.equals(JSONUtil.str(json, "package-id")))
679      throw new FHIRException("Package ids do not match in " + pu + ": " + id + " vs " + JSONUtil.str(json, "package-id"));
680    for (JsonElement e : json.getAsJsonArray("list")) {
681      JsonObject vo = (JsonObject) e;
682      if (v.equals(JSONUtil.str(vo, "version"))) {
683        aurl = Utilities.pathURL(JSONUtil.str(vo, "path"), "package.tgz");
684        String u = Utilities.pathURL(JSONUtil.str(vo, "path"), "package.tgz");
685        return new InputStreamWithSrc(fetchFromUrlSpecific(u, true), u, v);
686      }
687    }
688
689    return null;
690  }
691
692
693  // ---------- Current Build SubSystem --------------------------------------------------------------------------------------
694
695  private String fetchVersionTheOldWay(String id) throws IOException {
696    String url = getUrlForPackage(id);
697    if (url == null) {
698      try {
699        url = getPackageUrlFromBuildList(id);
700      } catch (Exception e) {
701        url = null;
702      }
703    }
704    if (url == null) {
705      throw new FHIRException("Unable to resolve package id " + id);
706    }
707    String pu = Utilities.pathURL(url, "package-list.json");
708    JsonObject json = fetchJson(pu);
709    if (!id.equals(JSONUtil.str(json, "package-id")))
710      throw new FHIRException("Package ids do not match in " + pu + ": " + id + " vs " + JSONUtil.str(json, "package-id"));
711    for (JsonElement e : json.getAsJsonArray("list")) {
712      JsonObject vo = (JsonObject) e;
713      if (JSONUtil.bool(vo, "current")) {
714        return JSONUtil.str(vo, "version");
715      }
716    }
717
718    return null;
719  }
720
721  private String getUrlForPackage(String id) {
722    if ("hl7.fhir.xver-extensions".equals(id)) {
723      return "http://fhir.org/packages/hl7.fhir.xver-extensions";
724    }
725    return null;
726  }
727
728  public List<String> listPackages() {
729    List<String> res = new ArrayList<>();
730    for (File f : new File(cacheFolder).listFiles()) {
731      if (f.isDirectory() && f.getName().contains("#")) {
732        res.add(f.getName());
733      }
734    }
735    return res;
736  }
737
738  /**
739   * if you don't provide and implementation of this interface, the PackageCacheManager will use the web directly.
740   * <p>
741   * You can use this interface to
742   *
743   * @author graha
744   */
745  public interface INetworkServices {
746
747    InputStream resolvePackage(String packageId, String version);
748  }
749
750  public interface CacheLockFunction<T> {
751    T get() throws IOException;
752  }
753
754  public class BuildRecordSorter implements Comparator<BuildRecord> {
755
756    @Override
757    public int compare(BuildRecord arg0, BuildRecord arg1) {
758      return arg1.date.compareTo(arg0.date);
759    }
760  }
761
762  public class BuildRecord {
763
764    private String url;
765    private String packageId;
766    private String repo;
767    private Date date;
768
769    public BuildRecord(String url, String packageId, String repo, Date date) {
770      super();
771      this.url = url;
772      this.packageId = packageId;
773      this.repo = repo;
774      this.date = date;
775    }
776
777    public String getUrl() {
778      return url;
779    }
780
781    public String getPackageId() {
782      return packageId;
783    }
784
785    public String getRepo() {
786      return repo;
787    }
788
789    public Date getDate() {
790      return date;
791    }
792
793
794  }
795
796
797
798  public class VersionHistory {
799    private String id;
800    private String canonical;
801    private String current;
802    private Map<String, String> versions = new HashMap<>();
803
804    public String getCanonical() {
805      return canonical;
806    }
807
808    public String getCurrent() {
809      return current;
810    }
811
812    public Map<String, String> getVersions() {
813      return versions;
814    }
815
816    public String getId() {
817      return id;
818    }
819  }
820
821  public class PackageEntry {
822
823    private byte[] bytes;
824    private String name;
825
826    public PackageEntry(String name) {
827      this.name = name;
828    }
829
830    public PackageEntry(String name, byte[] bytes) {
831      this.name = name;
832      this.bytes = bytes;
833    }
834  }
835
836  public class CacheLock {
837
838    private final File lockFile;
839
840    public CacheLock(String name) throws IOException {
841      this.lockFile = new File(cacheFolder, name + ".lock");
842      if (!lockFile.isFile()) {
843        TextFile.stringToFile("", lockFile);
844      }
845    }
846
847    public <T> T doWithLock(CacheLockFunction<T> f) throws FileNotFoundException, IOException {
848      try (FileChannel channel = new RandomAccessFile(lockFile, "rw").getChannel()) {
849        final FileLock fileLock = channel.lock();
850        T result = null;
851        try {
852          result = f.get();
853        } finally {
854          fileLock.release();
855        }
856        if (!lockFile.delete()) {
857          lockFile.deleteOnExit();
858        }
859        return result;
860      }
861    }
862  }
863
864
865
866
867
868
869//public List<String> getUrls() throws IOException {
870//  if (allUrls == null)
871//  {
872//    IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
873//    allUrls = new ArrayList<>();
874//    for (String s : ini.getPropertyNames("urls"))
875//      allUrls.add(ini.getStringProperty("urls", s));
876//    try {
877//      URL url = new URL("https://raw.githubusercontent.com/FHIR/ig-registry/master/fhir-ig-list.json?nocache=" + System.currentTimeMillis());
878//      HttpURLConnection connection = (HttpURLConnection) url.openConnection();
879//      connection.setRequestMethod("GET");
880//      InputStream json = connection.getInputStream();
881//      JsonObject packages = (JsonObject) new com.google.gson.JsonParser().parse(TextFile.streamToString(json));
882//      JsonArray guides = packages.getAsJsonArray("guides");
883//      for (JsonElement g : guides) {
884//        JsonObject gi = (JsonObject) g;
885//        if (gi.has("canonical"))
886//          if (!allUrls.contains(gi.get("canonical").getAsString()))
887//            allUrls.add(gi.get("canonical").getAsString());
888//      }
889//    } catch (Exception e) {
890//      System.out.println("Listing known Implementation Guides failed: "+e.getMessage());
891//    }
892//  }
893//  return allUrls;    
894//}
895}