001package org.hl7.fhir.utilities.npm;
002
003import java.io.*;
004
005import java.util.*;
006
007import javax.annotation.Nonnull;
008import javax.annotation.Nullable;
009
010import lombok.Getter;
011import lombok.Setter;
012import lombok.With;
013import lombok.extern.slf4j.Slf4j;
014import org.apache.commons.io.FileUtils;
015import org.hl7.fhir.exceptions.FHIRException;
016import org.hl7.fhir.utilities.*;
017import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
018import org.hl7.fhir.utilities.http.HTTPResult;
019import org.hl7.fhir.utilities.http.ManagedWebAccess;
020import org.hl7.fhir.utilities.json.model.JsonObject;
021import org.hl7.fhir.utilities.json.parser.JsonParser;
022import org.hl7.fhir.utilities.npm.PackageList.PackageListEntry;
023import org.hl7.fhir.utilities.settings.FhirSettings;
024import org.slf4j.Logger;
025import org.slf4j.LoggerFactory;
026
027/*
028  Copyright (c) 2011+, HL7, Inc.
029  All rights reserved.
030  
031  Redistribution and use in source and binary forms, with or without modification, 
032  are permitted provided that the following conditions are met:
033    
034   * Redistributions of source code must retain the above copyright notice, this 
035     list of conditions and the following disclaimer.
036   * Redistributions in binary form must reproduce the above copyright notice, 
037     this list of conditions and the following disclaimer in the documentation 
038     and/or other materials provided with the distribution.
039   * Neither the name of HL7 nor the names of its contributors may be used to 
040     endorse or promote products derived from this software without specific 
041     prior written permission.
042  
043  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
044  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
045  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 
046  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 
047  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 
048  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
049  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
050  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
051  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
052  POSSIBILITY OF SUCH DAMAGE.
053  
054 */
055
056/**
057 * This is a package cache manager implementation that uses a local disk cache
058 *
059 * <p>
060 * API:
061 * <p>
062 * constructor
063 * getPackageUrl
064 * getPackageId
065 * findPackageCache
066 * addPackageToCache
067 *
068 * @author Grahame Grieve
069 */
070@Slf4j
071public class FilesystemPackageCacheManager extends BasePackageCacheManager implements IPackageCacheManager {
072
073  private final FilesystemPackageCacheManagerLocks locks;
074
075  private final FilesystemPackageCacheManagerLocks.LockParameters lockParameters;
076
077  // When running in testing mode, some packages are provided from the test case repository rather than by the normal means
078  // the PackageProvider is responsible for this. if no package provider is defined, or it declines to handle the package, 
079  // then the normal means will be used
080  public interface IPackageProvider {
081    boolean handlesPackage(String id, String version);
082    InputStreamWithSrc provide(String id, String version) throws IOException;
083  }
084
085  private static IPackageProvider packageProvider;
086  public static final String PACKAGE_REGEX = "^[a-zA-Z][A-Za-z0-9\\_\\-]*(\\.[A-Za-z0-9\\_\\-]+)+$";
087  public static final String PACKAGE_VERSION_REGEX = "^[A-Za-z][A-Za-z0-9\\_\\-]*(\\.[A-Za-z0-9\\_\\-]+)+\\#[A-Za-z0-9\\-\\_\\$]+(\\.[A-Za-z0-9\\-\\_\\$]+)*$";
088  public static final String PACKAGE_VERSION_REGEX_OPT = "^[A-Za-z][A-Za-z0-9\\_\\-]*(\\.[A-Za-z0-9\\_\\-]+)+(\\#[A-Za-z0-9\\-\\_]+(\\.[A-Za-z0-9\\-\\_]+)*)?$";
089  private static final Logger ourLog = LoggerFactory.getLogger(FilesystemPackageCacheManager.class);
090  private static final String CACHE_VERSION = "3"; // second version - see wiki page
091
092  @Getter
093  private final CIBuildClient ciBuildClient;
094
095  @Nonnull
096  private final File cacheFolder;
097
098  private final List<NpmPackage> temporaryPackages = new ArrayList<>();
099
100  private boolean suppressErrors;
101
102  @Setter
103  @Getter
104  private boolean minimalMemory;
105
106  public static class Builder {
107
108    @Getter
109    private final File cacheFolder;
110
111    @With
112    @Getter
113    private final List<PackageServer> packageServers;
114
115    @With
116    @Getter
117    private final CIBuildClient ciBuildClient;
118
119    @With
120    @Getter
121    private final FilesystemPackageCacheManagerLocks.LockParameters lockParameters;
122
123    public Builder() throws IOException {
124      this.cacheFolder = getUserCacheFolder();
125      this.packageServers = getPackageServersFromFHIRSettings();
126      this.lockParameters = null;
127      this.ciBuildClient = new CIBuildClient();
128    }
129
130    private Builder(File cacheFolder, List<PackageServer> packageServers, CIBuildClient ciBuildClient, FilesystemPackageCacheManagerLocks.LockParameters lockParameters) {
131      this.cacheFolder = cacheFolder;
132      this.packageServers = packageServers;
133      this.ciBuildClient = ciBuildClient;
134      this.lockParameters = lockParameters;
135    }
136
137    private File getUserCacheFolder() throws IOException {
138      return ManagedFileAccess.file(Utilities.path(System.getProperty("user.home"), ".fhir", "packages"));
139    }
140
141    private List<PackageServer> getPackageServersFromFHIRSettings() {
142      List<PackageServer> packageServers = new ArrayList<>(getConfiguredServers());
143      if (!isIgnoreDefaultPackageServers()) {
144        packageServers.addAll(getDefaultServers());
145      }
146      return packageServers;
147    }
148
149    protected boolean isIgnoreDefaultPackageServers() {
150      return FhirSettings.isIgnoreDefaultPackageServers();
151    }
152
153    @Nonnull
154    protected List<PackageServer> getDefaultServers() {
155      return PackageServer.defaultServers();
156    }
157
158    protected List<PackageServer> getConfiguredServers() {
159      return PackageServer.getConfiguredServers();
160    }
161
162    public Builder withCacheFolder(String cacheFolderPath) throws IOException {
163      File cacheFolder = ManagedFileAccess.file(cacheFolderPath);
164      if (!cacheFolder.exists()) {
165        throw new FHIRException("The folder '" + cacheFolder + "' could not be found");
166      }
167      return new Builder(cacheFolder, this.packageServers, this.ciBuildClient, this.lockParameters);
168    }
169
170    public Builder withSystemCacheFolder() throws IOException {
171      final File systemCacheFolder;
172      if (Utilities.isWindows()) {
173        systemCacheFolder = ManagedFileAccess.file(Utilities.path(System.getenv("ProgramData"), ".fhir", "packages"));
174      } else {
175        systemCacheFolder = ManagedFileAccess.file(Utilities.path("/var", "lib", ".fhir", "packages"));
176      }
177      return new Builder(systemCacheFolder,  this.packageServers, this.ciBuildClient, this.lockParameters);
178    }
179
180    public Builder withTestingCacheFolder() throws IOException {
181      return new Builder(ManagedFileAccess.file(Utilities.path("[tmp]", ".fhir", "packages")), this.packageServers, this.ciBuildClient, this.lockParameters);
182    }
183
184    public FilesystemPackageCacheManager build() throws IOException {
185      final FilesystemPackageCacheManagerLocks locks;
186      try {
187        locks = FilesystemPackageCacheManagerLocks.getFilesystemPackageCacheManagerLocks(cacheFolder);
188      } catch (RuntimeException e) {
189        if (e.getCause() instanceof IOException) {
190          throw (IOException) e.getCause();
191        } else {
192          throw e;
193        }
194      }
195      return new FilesystemPackageCacheManager(cacheFolder, packageServers, ciBuildClient, locks, lockParameters);
196    }
197  }
198
199  private FilesystemPackageCacheManager(@Nonnull File cacheFolder, @Nonnull List<PackageServer> packageServers, CIBuildClient ciBuildClient, @Nonnull FilesystemPackageCacheManagerLocks locks, @Nullable FilesystemPackageCacheManagerLocks.LockParameters lockParameters) throws IOException {
200    super(packageServers);
201    this.cacheFolder = cacheFolder;
202    this.ciBuildClient = ciBuildClient;
203    this.locks = locks;
204    this.lockParameters = lockParameters;
205    prepareCacheFolder();
206  }
207
208  /**
209   * Check if the cache folder exists and is valid.
210   * <p>
211   * If it doesn't exist, create it.
212   * <p>
213   * If it does exist and isn't valid, delete it and create a new one.
214   * <p>
215   * If it does exist and is valid, just do some cleanup (delete temp download directories, etc.)
216   *
217   * @throws IOException if the cache folder can't be created
218   */
219  protected void prepareCacheFolder() throws IOException {
220    locks.getCacheLock().doWriteWithLock(() -> {
221
222      if (!(cacheFolder.exists())) {
223        FileUtilities.createDirectory(cacheFolder.getAbsolutePath());
224        createIniFile();
225      } else {
226        if (!iniFileExists()) {
227          createIniFile();
228        }
229        if (!isIniFileCurrentVersion()) {
230          clearCache();
231          createIniFile();
232        }
233        deleteOldTempDirectories();
234        cleanUpCorruptPackages();
235      }
236      return null;
237    });
238  }
239
240  /*
241    Look for .lock files that are not actively held by a process. If found, delete the lock file, and the package
242    referenced.
243   */
244  protected void cleanUpCorruptPackages() throws IOException {
245    for (File file : Objects.requireNonNull(cacheFolder.listFiles())) {
246      if (file.getName().endsWith(".lock")) {
247        if (locks.getCacheLock().canLockFileBeHeldByThisProcess(file)) {
248          String packageDirectoryName = file.getName().substring(0, file.getName().length() - 5);
249          log.info("Detected potential incomplete package installed in cache: " + packageDirectoryName + ". Attempting to delete");
250
251          File packageDirectory = ManagedFileAccess.file(Utilities.path(cacheFolder, packageDirectoryName));
252          if (packageDirectory.exists()) {
253            FileUtilities.clearDirectory(packageDirectory.getAbsolutePath());
254            packageDirectory.delete();
255          }
256          file.delete();
257          log.info("Deleted potential incomplete package: " + packageDirectoryName);
258        }
259      }
260    }
261  }
262
263  private boolean iniFileExists() throws IOException {
264    String iniPath = getPackagesIniPath();
265    File iniFile = ManagedFileAccess.file(iniPath);
266    return iniFile.exists();
267  }
268
269  private boolean isIniFileCurrentVersion() throws IOException {
270    String iniPath = getPackagesIniPath();
271    IniFile ini = new IniFile(iniPath);
272    String version = ini.getStringProperty("cache", "version");
273    return CACHE_VERSION.equals(version);
274  }
275
276  private void deleteOldTempDirectories() throws IOException {
277    for (File f : Objects.requireNonNull(cacheFolder.listFiles())) {
278      if (f.isDirectory() && UUIDUtilities.isValidUUID(f.getName())) {
279        FileUtilities.clearDirectory(f.getAbsolutePath());
280        f.delete();
281      }
282    }
283  }
284
285  public String getFolder() {
286    return cacheFolder.getAbsolutePath();
287  }
288
289  private NpmPackage loadPackageInfo(String path) throws IOException {
290    return minimalMemory ? NpmPackage.fromFolderMinimal(path, false) : NpmPackage.fromFolder(path, false);
291  }
292
293  private void clearCache() throws IOException {
294    for (File f : Objects.requireNonNull(cacheFolder.listFiles())) {
295      if (f.isDirectory()) {
296        FileUtilities.atomicDeleteDirectory(f.getAbsolutePath());
297
298      } else if (!f.getName().equals("packages.ini")) {
299        FileUtils.forceDelete(f);
300      }
301
302    }
303  }
304
305  private void createIniFile() throws IOException {
306    IniFile ini = new IniFile(getPackagesIniPath());
307    ini.setStringProperty("cache", "version", CACHE_VERSION, null);
308    ini.save();
309  }
310
311  private String getPackagesIniPath() throws IOException {
312    return Utilities.path(cacheFolder, "packages.ini");
313  }
314
315  private void checkValidVersionString(String version, String id) {
316    if (Utilities.noString(version)) {
317      throw new FHIRException("Cannot add package " + id + " to the package cache - a version must be provided");
318    }
319    if (version.startsWith("file:")) {
320      throw new FHIRException("Cannot add package " + id + " to the package cache - the version '" + version + "' is illegal in this context");
321    }
322    if (!(Utilities.existsInList(version, "current", "dev") || VersionUtilities.isSemVer(version))) {
323      throw new FHIRException("Cannot add package " + id + " to the package cache - the version '" + version + "' is illegal - must be a valid SemVer, or 'current' or 'dev'");
324    }
325  }
326
327  protected InputStreamWithSrc loadFromPackageServer(String id, String version) {
328    InputStreamWithSrc retVal = super.loadFromPackageServer(id, version);
329    if (retVal != null) {
330      return retVal;
331    }
332
333    retVal = super.loadFromPackageServer(id, VersionUtilities.getMajMin(version) + ".x");
334    if (retVal != null) {
335      return retVal;
336    }
337
338    // ok, well, we'll try the old way
339    return fetchTheOldWay(id, version);
340  }
341
342  public String getLatestVersion(String id, boolean milestonesOnly) throws IOException {
343    id = stripAlias(id);
344    for (PackageServer nextPackageServer : getPackageServers()) {
345      // special case:
346      if (!(Utilities.existsInList(id, CommonPackages.ID_PUBPACK, "hl7.terminology.r5") && PackageServer.SECONDARY_SERVER.equals(nextPackageServer.getUrl()))) {
347        PackageClient pc = new PackageClient(nextPackageServer);
348        try {
349          return pc.getLatestVersion(id, milestonesOnly);
350        } catch (IOException e) {
351          ourLog.info("Failed to determine latest version of package {} from server: {}", id, nextPackageServer.toString());
352        }
353      }
354    }
355    try {
356      return fetchVersionTheOldWay(id);
357    } catch (Exception e) {
358      ourLog.info("Failed to determine latest version of package {} from server: {}", id, "build.fhir.org");
359    }
360    // still here? use the latest version we previously found or at least, is in the cache
361
362    String version = getLatestVersionFromCache(id);
363    if (version != null) {
364      return version;
365    }
366    throw new FHIRException("Unable to find the last version for package " + id + ": no local copy, and no network access");
367  }
368
369  public String getLatestVersion(String id, String versionFilter) throws IOException {
370    id = stripAlias(id);
371    for (PackageServer nextPackageServer : getPackageServers()) {
372      // special case:
373      if (!(Utilities.existsInList(id, CommonPackages.ID_PUBPACK, "hl7.terminology.r5") && PackageServer.SECONDARY_SERVER.equals(nextPackageServer.getUrl()))) {
374        PackageClient pc = new PackageClient(nextPackageServer);
375        try {
376          return pc.getLatestVersion(id, versionFilter);
377        } catch (IOException e) {
378          ourLog.info("Failed to determine latest version of package {} from server: {}", id, nextPackageServer.toString());
379        }
380      }
381    }
382    try {
383      return fetchVersionTheOldWay(id, versionFilter);
384    } catch (Exception e) {
385      ourLog.info("Failed to determine latest version of package {} from server: {}", id, "build.fhir.org");
386    }
387    // still here? use the latest version we previously found or at least, is in the cache
388
389    String version = getLatestVersionFromCache(id, versionFilter);
390    if (version != null) {
391      return version;
392    }
393    throw new FHIRException("Unable to find the last version for package " + id + ": no local copy, and no network access");
394  }
395
396  public String getLatestVersionFromCache(String id) throws IOException {
397    id = stripAlias(id);
398    for (String f : Utilities.reverseSorted(cacheFolder.list())) {
399      File cf = ManagedFileAccess.file(Utilities.path(cacheFolder, f));
400      if (cf.isDirectory()) {
401        if (f.startsWith(id + "#")) {
402          String ver = f.substring(f.indexOf("#") + 1);
403          ourLog.info("Latest version of package {} found locally is {} - using that", id, ver);
404          return ver;
405        }
406      }
407    }
408    return null;
409  }
410
411  public String getLatestVersionFromCache(String id, String versionFilter) throws IOException {
412    id = stripAlias(id);
413    for (String f : Utilities.reverseSorted(cacheFolder.list())) {
414      File cf = ManagedFileAccess.file(Utilities.path(cacheFolder, f));
415      if (cf.isDirectory()) {
416        if (f.startsWith(id + "#")) {
417          String ver = f.substring(f.indexOf("#") + 1);
418          if (VersionUtilities.versionMatches(versionFilter, ver)) {
419            ourLog.info("Latest version of package {} found locally is {} - using that", id, ver);
420            return ver;
421          }
422        }
423      }
424    }
425    return null;
426  }
427
428  private NpmPackage loadPackageFromFile(String id, String folder) throws IOException {
429    File f = ManagedFileAccess.file(Utilities.path(folder, id));
430    if (!f.exists()) {
431      throw new FHIRException("Package '" + id + "  not found in folder " + folder);
432    }
433    if (!f.isDirectory()) {
434      throw new FHIRException("File for '" + id + "  found in folder " + folder + ", not a folder");
435    }
436    File fp = ManagedFileAccess.file(Utilities.path(folder, id, "package", "package.json"));
437    if (!fp.exists()) {
438      throw new FHIRException("Package '" + id + "  found in folder " + folder + ", but does not contain a package.json file in /package");
439    }
440    return NpmPackage.fromFolder(f.getAbsolutePath());
441  }
442
443  /**
444   * Clear the cache
445   *
446   * @throws IOException If the cache cannot be cleared
447   */
448  public void clear() throws IOException {
449    this.locks.getCacheLock().doWriteWithLock(() -> {
450      clearCache();
451      return null;
452    });
453  }
454
455  // ========================= Utilities ============================================================================
456
457  /**
458   * Remove a particular package from the cache
459   *
460   * @param id The id of the package to remove
461   * @param version The literal version of the package to remove. Values such as 'current' and 'dev' are not allowed.
462   * @throws IOException If the package cannot be removed
463   */
464  public void removePackage(String id, String version) throws IOException {
465    String sid = stripAlias(id);
466    locks.getPackageLock(sid + "#" + version).doWriteWithLock(() -> {
467
468      String f = Utilities.path(cacheFolder, sid + "#" + version);
469        File ff = ManagedFileAccess.file(f);
470        if (ff.exists()) {
471          FileUtilities.atomicDeleteDirectory(f);
472        }
473      return null;
474    }, lockParameters);
475  }
476
477  /**
478   * Load the identified package from the cache - if it exists
479   * <p/>
480   * This is for special purpose only (testing, control over speed of loading).
481   * <p/>
482   * Generally, use the loadPackage method
483   *
484   * @param id The id of the package to load
485   * @param version The version of the package to load. Values such as 'current' and 'dev' are allowed.
486   * @return The package, or null if it is not found
487   * @throws IOException If the package cannot be loaded
488   */
489  @Override
490  public NpmPackage loadPackageFromCacheOnly(String id, @Nullable String version) throws IOException {
491    id = stripAlias(id);
492
493    if (!Utilities.noString(version) && version.startsWith("file:")) {
494      return loadPackageFromFile(id, version.substring(5));
495    }
496
497    for (NpmPackage p : temporaryPackages) {
498      if (p.name().equals(id) && ("current".equals(version) || "dev".equals(version) || p.version().equals(version))) {
499        return p;
500      }
501      if (p.name().equals(id) && Utilities.noString(version)) {
502        return p;
503      }
504    }
505
506    String foundPackageFolder = findPackageFolder(id, version);
507    if (foundPackageFolder != null) {
508      NpmPackage foundPackage = locks.getPackageLock(foundPackageFolder).doReadWithLock(() -> {
509        String path = Utilities.path(cacheFolder, foundPackageFolder);
510        File directory = ManagedFileAccess.file(path);
511
512        /* Check if the directory still exists now that we have a read lock. findPackageFolder does no locking in order
513        to avoid locking every potential package directory, so it's possible that a package deletion has occurred.
514        * */
515        if (!directory.exists()) {
516          return null;
517        }
518        return loadPackageInfo(path);
519      }, lockParameters);
520      if (foundPackage != null) {
521        if (foundPackage.isIndexed()){
522          return foundPackage;
523        } else {
524          return locks.getPackageLock(foundPackageFolder).doWriteWithLock(() -> {
525            File directory = ManagedFileAccess.file(foundPackage.getPath());
526
527        /* Check if the directory still exists now that we have a write lock. findPackageFolder does no locking in order
528        to avoid locking every potential package directory, so it's possible that a package deletion has occurred.
529        * */
530            if (!directory.exists()) {
531              return null;
532            }
533
534            // Since another thread may have already indexed the package since our read, we need to check again
535            NpmPackage output = loadPackageInfo(foundPackage.getPath());
536            if (output.isIndexed()) {
537              return output;
538            }
539            String path = Utilities.path(cacheFolder, foundPackageFolder);
540            output.checkIndexed(path);
541            return output;
542          }, lockParameters);
543          }
544      }
545    }
546    if ("dev".equals(version))
547      return loadPackageFromCacheOnly(id, "current");
548    else
549      return null;
550  }
551
552  private String findPackageFolder(String id, String version) throws IOException {
553    String foundPackageFolder = null;
554    String foundVersion = null;
555    for (String currentPackageFolder : Utilities.reverseSorted(cacheFolder.list())) {
556      File cf = ManagedFileAccess.file(Utilities.path(cacheFolder, currentPackageFolder));
557      if (cf.isDirectory()) {
558        if (currentPackageFolder.equals(id + "#" + version) || (Utilities.noString(version) && currentPackageFolder.startsWith(id + "#"))) {
559          return currentPackageFolder;
560        }
561        if (version != null && !Utilities.existsInList(version, "current", "dev") && (VersionUtilities.isSemVerWithWildcards(version) || Utilities.charCount(version, '.') < 2) && currentPackageFolder.contains("#")) {
562          String[] parts = currentPackageFolder.split("#");
563          if (parts[0].equals(id) && VersionUtilities.isSemVer(parts[1]) && VersionUtilities.versionMatches((foundVersion != null ? foundVersion : version), parts[1])) {
564            foundVersion = parts[1];
565            foundPackageFolder = currentPackageFolder;
566          }
567        }
568      }
569    }
570    return foundPackageFolder;
571  }
572
573  /**
574   * Add an already fetched package to the cache
575   */
576  @Override
577  public NpmPackage addPackageToCache(String id, final String version, final InputStream packageTgzInputStream, final String sourceDesc) throws IOException {
578    String sid = stripAlias(id);
579    checkValidVersionString(version, sid);
580    return locks.getPackageLock(sid + "#" + version).doWriteWithLock(() -> {
581
582      String tempDir = Utilities.generateUniqueRandomUUIDPath(cacheFolder.getAbsolutePath());
583
584      NpmPackage extractedNpm = NpmPackage.extractFromTgz(packageTgzInputStream, sourceDesc, tempDir, minimalMemory);
585
586      log.info("");
587      log.info("Installing " + sid + "#" + version);
588
589      if ((extractedNpm.name() != null && sid != null && !sid.equalsIgnoreCase(extractedNpm.name()) && !sid.equalsIgnoreCase(extractedNpm.name()+"."+VersionUtilities.getNameForVersion(extractedNpm.fhirVersion())))) {
590        if (!suppressErrors && (!sid.equals("hl7.fhir.r5.core") && !sid.equals("hl7.fhir.us.immds"))) {// temporary work around
591          throw new IOException("Attempt to import a mis-identified package. Expected " + sid + ", got " + extractedNpm.name());
592        }
593      }
594
595      NpmPackage npmPackage;
596      String packageRoot = Utilities.path(cacheFolder, sid + "#" + version);
597      try {
598        if (!sid.equals(extractedNpm.getNpm().asString("name")) || !version.equals(extractedNpm.getNpm().asString("version"))) {
599          if (!sid.equals(extractedNpm.getNpm().asString("name"))) {
600            extractedNpm.getNpm().add("original-name", extractedNpm.getNpm().asString("name"));
601            extractedNpm.getNpm().remove("name");
602            extractedNpm.getNpm().add("name", sid);
603          }
604          if (!version.equals(extractedNpm.getNpm().asString("version"))) {
605            extractedNpm.getNpm().add("original-version", extractedNpm.getNpm().asString("version"));
606            extractedNpm.getNpm().remove("version");
607            extractedNpm.getNpm().add("version", version);
608          }
609          FileUtilities.stringToFile(JsonParser.compose(extractedNpm.getNpm(), true), Utilities.path(tempDir, "package", "package.json"));
610        }
611
612        final NpmPackage tempPackage = loadPackageInfo(tempDir);
613        if (tempPackage != null && !tempPackage.isIndexed()) {
614          tempPackage.checkIndexed(packageRoot);
615        }
616
617        if (!ManagedFileAccess.file(packageRoot).exists() || Utilities.existsInList(version, "current", "dev")) {
618          FileUtilities.createDirectory(packageRoot);
619          try {
620            FileUtilities.clearDirectory(packageRoot);
621          } catch (Throwable t) {
622            log.info("Unable to clear directory: " + packageRoot + ": " + t.getMessage() + " - this may cause problems later");
623          }
624          FileUtilities.renameDirectory(tempDir, packageRoot);
625
626          npmPackage = loadPackageInfo(packageRoot);
627
628          log.info(" done.");
629        } else {
630          FileUtilities.clearDirectory(tempDir);
631          ManagedFileAccess.file(tempDir).delete();
632        }
633        npmPackage = loadPackageInfo(packageRoot);
634
635      } catch (Exception e) {
636        try {
637          // don't leave a half extracted package behind
638          log.info("Clean up package " + packageRoot + " because installation failed: " + e.getMessage());
639          e.printStackTrace();
640          FileUtilities.clearDirectory(packageRoot);
641          ManagedFileAccess.file(packageRoot).delete();
642        } catch (Exception ignored) {
643          // nothing
644        }
645        throw e;
646      }
647      return npmPackage;
648    }, lockParameters);
649  }
650
651  @Override
652  public String getPackageUrl(String packageId) throws IOException {
653    packageId = stripAlias(packageId);
654    String result = super.getPackageUrl(packageId);
655    if (result == null) {
656      result = ciBuildClient.getPackageUrl(packageId);
657    }
658    return result;
659  }
660
661  /**
662   * do not use this in minimal memory mode
663   * @param packagesFolder
664   * @throws IOException
665   */
666  public void loadFromFolder(String packagesFolder) throws IOException {
667    assert !minimalMemory;
668
669    File[] files = ManagedFileAccess.file(packagesFolder).listFiles();
670    if (files != null) {
671      for (File f : files) {
672        if (f.getName().endsWith(".tgz")) {
673          FileInputStream fs = ManagedFileAccess.inStream(f);
674          try {
675            temporaryPackages.add(NpmPackage.fromPackage(fs));
676          } finally {
677            fs.close();
678          }
679        }
680      }
681    }
682  }
683
684  @Override
685  public NpmPackage loadPackage(String id, String version) throws FHIRException, IOException {
686    id = stripAlias(id);
687    //ok, try to resolve locally
688    if (!Utilities.noString(version) && version.startsWith("file:")) {
689      return loadPackageFromFile(id, version.substring(5));
690    }
691
692    if (version == null && id.contains("#")) {
693      version = id.substring(id.indexOf("#") + 1);
694      id = id.substring(0, id.indexOf("#"));
695    }
696
697    if (version == null) {
698      try {
699        version = getLatestVersion(id, false);
700      } catch (Exception e) {
701        version = null;
702      }
703    }
704    NpmPackage p = loadPackageFromCacheOnly(id, version);
705    if (p != null) {
706      if ("current".equals(version)) {
707        p = nullIfNotCurrentPackage(id, p);
708      }
709      if (p != null)
710        return p;
711    }
712
713    if ("dev".equals(version)) {
714      p = loadPackageFromCacheOnly(id, "current");
715      p = nullIfNotCurrentPackage(id, p);
716      if (p != null)
717        return p;
718      version = "current";
719    }
720
721    log.info("Installing " + id + "#" + (version == null ? "?" : version) + " to the package cache");
722    log.info("  Fetching:");
723
724    // nup, don't have it locally (or it's expired)
725    FilesystemPackageCacheManager.InputStreamWithSrc source;
726    if (false && packageProvider != null && packageProvider.handlesPackage(id, version)) {
727      source = packageProvider.provide(id, version);
728    } else if (Utilities.isAbsoluteUrl(version)) {
729      source = fetchSourceFromUrlSpecific(version);
730    } else if ("current".equals(version) || (version != null && version.startsWith("current$"))) {
731      // special case - fetch from ci-build server
732      source = ciBuildClient.loadFromCIBuild(id, version.startsWith("current$") ? version.substring(8) : null);
733    } else {
734      source = loadFromPackageServer(id, version);
735    }
736    if (source == null) {
737      throw new FHIRException("Unable to find package " + id + "#" + version);
738    }
739    return addPackageToCache(id, source.version, source.stream, source.url);
740  }
741
742  private InputStreamWithSrc fetchSourceFromUrlSpecific(String url) {
743    return new InputStreamWithSrc(fetchFromUrlSpecific(url, false), url, "current");
744  }
745
746  private InputStream fetchFromUrlSpecific(String source, boolean optional) throws FHIRException {
747    try {
748      HTTPResult res = ManagedWebAccess.get(Arrays.asList("web"), source);
749      res.checkThrowException();
750      return new ByteArrayInputStream(res.getContent());
751    } catch (Exception e) {
752      if (optional)
753        return null;
754      else
755        throw new FHIRException("Unable to fetch: " + e.getMessage(), e);
756    }
757  }
758
759
760
761  @Override
762  public String getPackageId(String canonicalUrl) throws IOException {
763    String retVal = findCanonicalInLocalCache(canonicalUrl);
764
765    if (retVal == null) {
766      retVal = super.getPackageId(canonicalUrl);
767    }
768
769    if (retVal == null) {
770      retVal = ciBuildClient.getPackageId(canonicalUrl);
771    }
772
773    return retVal;
774  }
775
776
777  public String findCanonicalInLocalCache(String canonicalUrl) {
778    try {
779      for (String pf : listPackages()) {
780        if (ManagedFileAccess.file(Utilities.path(cacheFolder, pf, "package", "package.json")).exists()) {
781          JsonObject npm = JsonParser.parseObjectFromFile(Utilities.path(cacheFolder, pf, "package", "package.json"));
782          if (canonicalUrl.equals(npm.asString("canonical"))) {
783            return npm.asString("name");
784          }
785        }
786      }
787    } catch (IOException e) {
788    }
789    return null;
790  }
791
792  // ========================= Package Mgmt API =======================================================================
793
794  /**
795   * Checks https://
796   *
797   * @param id
798   * @param npmPackage
799   * @return
800   */
801  private NpmPackage nullIfNotCurrentPackage(String id, NpmPackage npmPackage) {
802    // special case: current versions roll over, and we have to check their currency
803    try {
804      return ciBuildClient.isCurrent(id, npmPackage) ? npmPackage : null;
805    } catch (Exception e) {
806      log.info("Unable to check package currency: " + id + ": " + id);
807    }
808    return npmPackage;
809  }
810
811  // ----- the old way, from before package server, while everything gets onto the package server
812  private InputStreamWithSrc fetchTheOldWay(String id, String v) {
813    String url = getUrlForPackage(id);
814    if (url == null) {
815      try {
816        url = ciBuildClient.getPackageUrl(id);
817      } catch (Exception ignored) {
818
819      }
820    }
821    if (url == null) {
822      throw new FHIRException("Unable to resolve package id " + id + "#" + v);
823    }
824    if (url.contains("/ImplementationGuide/")) {
825      url = url.substring(0, url.indexOf("/ImplementationGuide/"));
826    }
827    String pu = Utilities.pathURL(url, "package-list.json");
828
829    PackageList pl;
830    try {
831      pl = PackageList.fromUrl(pu);
832    } catch (Exception e) {
833      String pv = Utilities.pathURL(url, v, "package.tgz");
834      try {
835        return new InputStreamWithSrc(fetchFromUrlSpecific(pv, false), pv, v);
836      } catch (Exception e1) {
837        throw new FHIRException("Error fetching package directly (" + pv + "), or fetching package list for " + id + " from " + pu + ": " + e1.getMessage(), e1);
838      }
839    }
840    if (!id.equals(pl.pid()))
841      throw new FHIRException("Package ids do not match in " + pu + ": " + id + " vs " + pl.pid());
842    for (PackageListEntry vo : pl.versions()) {
843      if (v.equals(vo.version())) {
844
845        String u = Utilities.pathURL(vo.path(), "package.tgz");
846        return new InputStreamWithSrc(fetchFromUrlSpecific(u, true), u, v);
847      }
848    }
849
850    return null;
851  }
852
853
854  // ---------- Current Build SubSystem --------------------------------------------------------------------------------------
855
856  private String fetchVersionTheOldWay(String id) throws IOException {
857    String url = getUrlForPackage(id);
858    if (url == null) {
859      try {
860        url = ciBuildClient.getPackageUrl(id);
861      } catch (Exception e) {
862        url = null;
863      }
864    }
865    if (url == null) {
866      throw new FHIRException("Unable to resolve package id " + id);
867    }
868    PackageList pl = PackageList.fromUrl(Utilities.pathURL(url, "package-list.json"));
869    if (!id.equals(pl.pid()))
870      throw new FHIRException("Package ids do not match in " + pl.source() + ": " + id + " vs " + pl.pid());
871    for (PackageListEntry vo : pl.versions()) {
872      if (vo.current()) {
873        return vo.version();
874      }
875    }
876
877    return null;
878  }
879
880  private String fetchVersionTheOldWay(String id, String versionSpec) throws IOException {
881    String url = getUrlForPackage(id);
882    if (url == null) {
883      try {
884        url = ciBuildClient.getPackageUrl(id);
885      } catch (Exception e) {
886        url = null;
887      }
888    }
889    if (url == null) {
890      throw new FHIRException("Unable to resolve package id " + id);
891    }
892    PackageList pl = PackageList.fromUrl(Utilities.pathURL(url, "package-list.json"));
893    if (!id.equals(pl.pid()))
894      throw new FHIRException("Package ids do not match in " + pl.source() + ": " + id + " vs " + pl.pid());
895    for (PackageListEntry vo : pl.versions()) {
896      if (VersionUtilities.versionMatches(versionSpec, vo.version())) {
897        return vo.version();
898      }
899    }
900
901    return null;
902  }
903
904  private String getUrlForPackage(String id) {
905    if (CommonPackages.ID_XVER.equals(id)) {
906      return "https://fhir.org/packages/hl7.fhir.xver-extensions";
907    }
908    return null;
909  }
910
911  public List<String> listPackages() {
912    List<String> res = new ArrayList<>();
913    for (File f : cacheFolder.listFiles()) {
914      if (f.isDirectory() && f.getName().contains("#")) {
915        res.add(f.getName());
916      }
917    }
918    return res;
919  }
920
921  public interface CacheLockFunction<T> {
922    T get() throws IOException;
923  }
924
925
926
927
928
929  public boolean packageExists(String id, String ver) throws IOException {
930    id = stripAlias(id);
931    if (packageInstalled(id, ver)) {
932      return true;
933    }
934    for (PackageServer s : getPackageServers()) {
935      if (new PackageClient(s).exists(id, ver)) {
936        return true;
937      }
938    }
939    return false;
940  }
941
942  public boolean packageInstalled(String id, String version) {
943    id = stripAlias(id);
944    for (NpmPackage p : temporaryPackages) {
945      if (p.name().equals(id) && ("current".equals(version) || "dev".equals(version) || p.version().equals(version))) {
946        return true;
947      }
948      if (p.name().equals(id) && Utilities.noString(version)) {
949        return true;
950      }
951    }
952
953    for (String f : Utilities.sorted(cacheFolder.list())) {
954      if (f.equals(id + "#" + version) || (Utilities.noString(version) && f.startsWith(id + "#"))) {
955        return true;
956      }
957    }
958    if ("dev".equals(version))
959      return packageInstalled(id, "current");
960    else
961      return false;
962  }
963
964  public boolean isSuppressErrors() {
965    return suppressErrors;
966  }
967
968  public void setSuppressErrors(boolean suppressErrors) {
969    this.suppressErrors = suppressErrors;
970  }
971
972  public static IPackageProvider getPackageProvider() {
973    return packageProvider;
974  }
975
976  public static void setPackageProvider(IPackageProvider packageProvider) {
977    FilesystemPackageCacheManager.packageProvider = packageProvider;
978  }
979
980
981}