001package org.hl7.fhir.utilities.npm; 002 003import java.io.*; 004 005import java.util.*; 006 007import javax.annotation.Nonnull; 008import javax.annotation.Nullable; 009 010import lombok.Getter; 011import lombok.Setter; 012import lombok.With; 013import lombok.extern.slf4j.Slf4j; 014import org.apache.commons.io.FileUtils; 015import org.hl7.fhir.exceptions.FHIRException; 016import org.hl7.fhir.utilities.*; 017import org.hl7.fhir.utilities.filesystem.ManagedFileAccess; 018import org.hl7.fhir.utilities.http.HTTPResult; 019import org.hl7.fhir.utilities.http.ManagedWebAccess; 020import org.hl7.fhir.utilities.json.model.JsonObject; 021import org.hl7.fhir.utilities.json.parser.JsonParser; 022import org.hl7.fhir.utilities.npm.PackageList.PackageListEntry; 023import org.hl7.fhir.utilities.settings.FhirSettings; 024import org.slf4j.Logger; 025import org.slf4j.LoggerFactory; 026 027/* 028 Copyright (c) 2011+, HL7, Inc. 029 All rights reserved. 030 031 Redistribution and use in source and binary forms, with or without modification, 032 are permitted provided that the following conditions are met: 033 034 * Redistributions of source code must retain the above copyright notice, this 035 list of conditions and the following disclaimer. 036 * Redistributions in binary form must reproduce the above copyright notice, 037 this list of conditions and the following disclaimer in the documentation 038 and/or other materials provided with the distribution. 039 * Neither the name of HL7 nor the names of its contributors may be used to 040 endorse or promote products derived from this software without specific 041 prior written permission. 042 043 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 044 ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 045 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 046 IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 047 INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 048 NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 049 PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 050 WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 051 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 052 POSSIBILITY OF SUCH DAMAGE. 053 054 */ 055 056/** 057 * This is a package cache manager implementation that uses a local disk cache 058 * 059 * <p> 060 * API: 061 * <p> 062 * constructor 063 * getPackageUrl 064 * getPackageId 065 * findPackageCache 066 * addPackageToCache 067 * 068 * @author Grahame Grieve 069 */ 070@Slf4j 071public class FilesystemPackageCacheManager extends BasePackageCacheManager implements IPackageCacheManager { 072 073 private final FilesystemPackageCacheManagerLocks locks; 074 075 private final FilesystemPackageCacheManagerLocks.LockParameters lockParameters; 076 077 // When running in testing mode, some packages are provided from the test case repository rather than by the normal means 078 // the PackageProvider is responsible for this. if no package provider is defined, or it declines to handle the package, 079 // then the normal means will be used 080 public interface IPackageProvider { 081 boolean handlesPackage(String id, String version); 082 InputStreamWithSrc provide(String id, String version) throws IOException; 083 } 084 085 private static IPackageProvider packageProvider; 086 public static final String PACKAGE_REGEX = "^[a-zA-Z][A-Za-z0-9\\_\\-]*(\\.[A-Za-z0-9\\_\\-]+)+$"; 087 public static final String PACKAGE_VERSION_REGEX = "^[A-Za-z][A-Za-z0-9\\_\\-]*(\\.[A-Za-z0-9\\_\\-]+)+\\#[A-Za-z0-9\\-\\_\\$]+(\\.[A-Za-z0-9\\-\\_\\$]+)*$"; 088 public static final String PACKAGE_VERSION_REGEX_OPT = "^[A-Za-z][A-Za-z0-9\\_\\-]*(\\.[A-Za-z0-9\\_\\-]+)+(\\#[A-Za-z0-9\\-\\_]+(\\.[A-Za-z0-9\\-\\_]+)*)?$"; 089 private static final Logger ourLog = LoggerFactory.getLogger(FilesystemPackageCacheManager.class); 090 private static final String CACHE_VERSION = "3"; // second version - see wiki page 091 092 @Getter 093 private final CIBuildClient ciBuildClient; 094 095 @Nonnull 096 private final File cacheFolder; 097 098 private final List<NpmPackage> temporaryPackages = new ArrayList<>(); 099 100 private boolean suppressErrors; 101 102 @Setter 103 @Getter 104 private boolean minimalMemory; 105 106 public static class Builder { 107 108 @Getter 109 private final File cacheFolder; 110 111 @With 112 @Getter 113 private final List<PackageServer> packageServers; 114 115 @With 116 @Getter 117 private final CIBuildClient ciBuildClient; 118 119 @With 120 @Getter 121 private final FilesystemPackageCacheManagerLocks.LockParameters lockParameters; 122 123 public Builder() throws IOException { 124 this.cacheFolder = getUserCacheFolder(); 125 this.packageServers = getPackageServersFromFHIRSettings(); 126 this.lockParameters = null; 127 this.ciBuildClient = new CIBuildClient(); 128 } 129 130 private Builder(File cacheFolder, List<PackageServer> packageServers, CIBuildClient ciBuildClient, FilesystemPackageCacheManagerLocks.LockParameters lockParameters) { 131 this.cacheFolder = cacheFolder; 132 this.packageServers = packageServers; 133 this.ciBuildClient = ciBuildClient; 134 this.lockParameters = lockParameters; 135 } 136 137 private File getUserCacheFolder() throws IOException { 138 return ManagedFileAccess.file(Utilities.path(System.getProperty("user.home"), ".fhir", "packages")); 139 } 140 141 private List<PackageServer> getPackageServersFromFHIRSettings() { 142 List<PackageServer> packageServers = new ArrayList<>(getConfiguredServers()); 143 if (!isIgnoreDefaultPackageServers()) { 144 packageServers.addAll(getDefaultServers()); 145 } 146 return packageServers; 147 } 148 149 protected boolean isIgnoreDefaultPackageServers() { 150 return FhirSettings.isIgnoreDefaultPackageServers(); 151 } 152 153 @Nonnull 154 protected List<PackageServer> getDefaultServers() { 155 return PackageServer.defaultServers(); 156 } 157 158 protected List<PackageServer> getConfiguredServers() { 159 return PackageServer.getConfiguredServers(); 160 } 161 162 public Builder withCacheFolder(String cacheFolderPath) throws IOException { 163 File cacheFolder = ManagedFileAccess.file(cacheFolderPath); 164 if (!cacheFolder.exists()) { 165 throw new FHIRException("The folder '" + cacheFolder + "' could not be found"); 166 } 167 return new Builder(cacheFolder, this.packageServers, this.ciBuildClient, this.lockParameters); 168 } 169 170 public Builder withSystemCacheFolder() throws IOException { 171 final File systemCacheFolder; 172 if (Utilities.isWindows()) { 173 systemCacheFolder = ManagedFileAccess.file(Utilities.path(System.getenv("ProgramData"), ".fhir", "packages")); 174 } else { 175 systemCacheFolder = ManagedFileAccess.file(Utilities.path("/var", "lib", ".fhir", "packages")); 176 } 177 return new Builder(systemCacheFolder, this.packageServers, this.ciBuildClient, this.lockParameters); 178 } 179 180 public Builder withTestingCacheFolder() throws IOException { 181 return new Builder(ManagedFileAccess.file(Utilities.path("[tmp]", ".fhir", "packages")), this.packageServers, this.ciBuildClient, this.lockParameters); 182 } 183 184 public FilesystemPackageCacheManager build() throws IOException { 185 final FilesystemPackageCacheManagerLocks locks; 186 try { 187 locks = FilesystemPackageCacheManagerLocks.getFilesystemPackageCacheManagerLocks(cacheFolder); 188 } catch (RuntimeException e) { 189 if (e.getCause() instanceof IOException) { 190 throw (IOException) e.getCause(); 191 } else { 192 throw e; 193 } 194 } 195 return new FilesystemPackageCacheManager(cacheFolder, packageServers, ciBuildClient, locks, lockParameters); 196 } 197 } 198 199 private FilesystemPackageCacheManager(@Nonnull File cacheFolder, @Nonnull List<PackageServer> packageServers, CIBuildClient ciBuildClient, @Nonnull FilesystemPackageCacheManagerLocks locks, @Nullable FilesystemPackageCacheManagerLocks.LockParameters lockParameters) throws IOException { 200 super(packageServers); 201 this.cacheFolder = cacheFolder; 202 this.ciBuildClient = ciBuildClient; 203 this.locks = locks; 204 this.lockParameters = lockParameters; 205 prepareCacheFolder(); 206 } 207 208 /** 209 * Check if the cache folder exists and is valid. 210 * <p> 211 * If it doesn't exist, create it. 212 * <p> 213 * If it does exist and isn't valid, delete it and create a new one. 214 * <p> 215 * If it does exist and is valid, just do some cleanup (delete temp download directories, etc.) 216 * 217 * @throws IOException if the cache folder can't be created 218 */ 219 protected void prepareCacheFolder() throws IOException { 220 locks.getCacheLock().doWriteWithLock(() -> { 221 222 if (!(cacheFolder.exists())) { 223 FileUtilities.createDirectory(cacheFolder.getAbsolutePath()); 224 createIniFile(); 225 } else { 226 if (!iniFileExists()) { 227 createIniFile(); 228 } 229 if (!isIniFileCurrentVersion()) { 230 clearCache(); 231 createIniFile(); 232 } 233 deleteOldTempDirectories(); 234 cleanUpCorruptPackages(); 235 } 236 return null; 237 }); 238 } 239 240 /* 241 Look for .lock files that are not actively held by a process. If found, delete the lock file, and the package 242 referenced. 243 */ 244 protected void cleanUpCorruptPackages() throws IOException { 245 for (File file : Objects.requireNonNull(cacheFolder.listFiles())) { 246 if (file.getName().endsWith(".lock")) { 247 if (locks.getCacheLock().canLockFileBeHeldByThisProcess(file)) { 248 String packageDirectoryName = file.getName().substring(0, file.getName().length() - 5); 249 log.info("Detected potential incomplete package installed in cache: " + packageDirectoryName + ". Attempting to delete"); 250 251 File packageDirectory = ManagedFileAccess.file(Utilities.path(cacheFolder, packageDirectoryName)); 252 if (packageDirectory.exists()) { 253 FileUtilities.clearDirectory(packageDirectory.getAbsolutePath()); 254 packageDirectory.delete(); 255 } 256 file.delete(); 257 log.info("Deleted potential incomplete package: " + packageDirectoryName); 258 } 259 } 260 } 261 } 262 263 private boolean iniFileExists() throws IOException { 264 String iniPath = getPackagesIniPath(); 265 File iniFile = ManagedFileAccess.file(iniPath); 266 return iniFile.exists(); 267 } 268 269 private boolean isIniFileCurrentVersion() throws IOException { 270 String iniPath = getPackagesIniPath(); 271 IniFile ini = new IniFile(iniPath); 272 String version = ini.getStringProperty("cache", "version"); 273 return CACHE_VERSION.equals(version); 274 } 275 276 private void deleteOldTempDirectories() throws IOException { 277 for (File f : Objects.requireNonNull(cacheFolder.listFiles())) { 278 if (f.isDirectory() && UUIDUtilities.isValidUUID(f.getName())) { 279 FileUtilities.clearDirectory(f.getAbsolutePath()); 280 f.delete(); 281 } 282 } 283 } 284 285 public String getFolder() { 286 return cacheFolder.getAbsolutePath(); 287 } 288 289 private NpmPackage loadPackageInfo(String path) throws IOException { 290 return minimalMemory ? NpmPackage.fromFolderMinimal(path, false) : NpmPackage.fromFolder(path, false); 291 } 292 293 private void clearCache() throws IOException { 294 for (File f : Objects.requireNonNull(cacheFolder.listFiles())) { 295 if (f.isDirectory()) { 296 FileUtilities.atomicDeleteDirectory(f.getAbsolutePath()); 297 298 } else if (!f.getName().equals("packages.ini")) { 299 FileUtils.forceDelete(f); 300 } 301 302 } 303 } 304 305 private void createIniFile() throws IOException { 306 IniFile ini = new IniFile(getPackagesIniPath()); 307 ini.setStringProperty("cache", "version", CACHE_VERSION, null); 308 ini.save(); 309 } 310 311 private String getPackagesIniPath() throws IOException { 312 return Utilities.path(cacheFolder, "packages.ini"); 313 } 314 315 private void checkValidVersionString(String version, String id) { 316 if (Utilities.noString(version)) { 317 throw new FHIRException("Cannot add package " + id + " to the package cache - a version must be provided"); 318 } 319 if (version.startsWith("file:")) { 320 throw new FHIRException("Cannot add package " + id + " to the package cache - the version '" + version + "' is illegal in this context"); 321 } 322 if (!(Utilities.existsInList(version, "current", "dev") || VersionUtilities.isSemVer(version))) { 323 throw new FHIRException("Cannot add package " + id + " to the package cache - the version '" + version + "' is illegal - must be a valid SemVer, or 'current' or 'dev'"); 324 } 325 } 326 327 protected InputStreamWithSrc loadFromPackageServer(String id, String version) { 328 InputStreamWithSrc retVal = super.loadFromPackageServer(id, version); 329 if (retVal != null) { 330 return retVal; 331 } 332 333 retVal = super.loadFromPackageServer(id, VersionUtilities.getMajMin(version) + ".x"); 334 if (retVal != null) { 335 return retVal; 336 } 337 338 // ok, well, we'll try the old way 339 return fetchTheOldWay(id, version); 340 } 341 342 public String getLatestVersion(String id, boolean milestonesOnly) throws IOException { 343 id = stripAlias(id); 344 for (PackageServer nextPackageServer : getPackageServers()) { 345 // special case: 346 if (!(Utilities.existsInList(id, CommonPackages.ID_PUBPACK, "hl7.terminology.r5") && PackageServer.SECONDARY_SERVER.equals(nextPackageServer.getUrl()))) { 347 PackageClient pc = new PackageClient(nextPackageServer); 348 try { 349 return pc.getLatestVersion(id, milestonesOnly); 350 } catch (IOException e) { 351 ourLog.info("Failed to determine latest version of package {} from server: {}", id, nextPackageServer.toString()); 352 } 353 } 354 } 355 try { 356 return fetchVersionTheOldWay(id); 357 } catch (Exception e) { 358 ourLog.info("Failed to determine latest version of package {} from server: {}", id, "build.fhir.org"); 359 } 360 // still here? use the latest version we previously found or at least, is in the cache 361 362 String version = getLatestVersionFromCache(id); 363 if (version != null) { 364 return version; 365 } 366 throw new FHIRException("Unable to find the last version for package " + id + ": no local copy, and no network access"); 367 } 368 369 public String getLatestVersion(String id, String versionFilter) throws IOException { 370 id = stripAlias(id); 371 for (PackageServer nextPackageServer : getPackageServers()) { 372 // special case: 373 if (!(Utilities.existsInList(id, CommonPackages.ID_PUBPACK, "hl7.terminology.r5") && PackageServer.SECONDARY_SERVER.equals(nextPackageServer.getUrl()))) { 374 PackageClient pc = new PackageClient(nextPackageServer); 375 try { 376 return pc.getLatestVersion(id, versionFilter); 377 } catch (IOException e) { 378 ourLog.info("Failed to determine latest version of package {} from server: {}", id, nextPackageServer.toString()); 379 } 380 } 381 } 382 try { 383 return fetchVersionTheOldWay(id, versionFilter); 384 } catch (Exception e) { 385 ourLog.info("Failed to determine latest version of package {} from server: {}", id, "build.fhir.org"); 386 } 387 // still here? use the latest version we previously found or at least, is in the cache 388 389 String version = getLatestVersionFromCache(id, versionFilter); 390 if (version != null) { 391 return version; 392 } 393 throw new FHIRException("Unable to find the last version for package " + id + ": no local copy, and no network access"); 394 } 395 396 public String getLatestVersionFromCache(String id) throws IOException { 397 id = stripAlias(id); 398 for (String f : Utilities.reverseSorted(cacheFolder.list())) { 399 File cf = ManagedFileAccess.file(Utilities.path(cacheFolder, f)); 400 if (cf.isDirectory()) { 401 if (f.startsWith(id + "#")) { 402 String ver = f.substring(f.indexOf("#") + 1); 403 ourLog.info("Latest version of package {} found locally is {} - using that", id, ver); 404 return ver; 405 } 406 } 407 } 408 return null; 409 } 410 411 public String getLatestVersionFromCache(String id, String versionFilter) throws IOException { 412 id = stripAlias(id); 413 for (String f : Utilities.reverseSorted(cacheFolder.list())) { 414 File cf = ManagedFileAccess.file(Utilities.path(cacheFolder, f)); 415 if (cf.isDirectory()) { 416 if (f.startsWith(id + "#")) { 417 String ver = f.substring(f.indexOf("#") + 1); 418 if (VersionUtilities.versionMatches(versionFilter, ver)) { 419 ourLog.info("Latest version of package {} found locally is {} - using that", id, ver); 420 return ver; 421 } 422 } 423 } 424 } 425 return null; 426 } 427 428 private NpmPackage loadPackageFromFile(String id, String folder) throws IOException { 429 File f = ManagedFileAccess.file(Utilities.path(folder, id)); 430 if (!f.exists()) { 431 throw new FHIRException("Package '" + id + " not found in folder " + folder); 432 } 433 if (!f.isDirectory()) { 434 throw new FHIRException("File for '" + id + " found in folder " + folder + ", not a folder"); 435 } 436 File fp = ManagedFileAccess.file(Utilities.path(folder, id, "package", "package.json")); 437 if (!fp.exists()) { 438 throw new FHIRException("Package '" + id + " found in folder " + folder + ", but does not contain a package.json file in /package"); 439 } 440 return NpmPackage.fromFolder(f.getAbsolutePath()); 441 } 442 443 /** 444 * Clear the cache 445 * 446 * @throws IOException If the cache cannot be cleared 447 */ 448 public void clear() throws IOException { 449 this.locks.getCacheLock().doWriteWithLock(() -> { 450 clearCache(); 451 return null; 452 }); 453 } 454 455 // ========================= Utilities ============================================================================ 456 457 /** 458 * Remove a particular package from the cache 459 * 460 * @param id The id of the package to remove 461 * @param version The literal version of the package to remove. Values such as 'current' and 'dev' are not allowed. 462 * @throws IOException If the package cannot be removed 463 */ 464 public void removePackage(String id, String version) throws IOException { 465 String sid = stripAlias(id); 466 locks.getPackageLock(sid + "#" + version).doWriteWithLock(() -> { 467 468 String f = Utilities.path(cacheFolder, sid + "#" + version); 469 File ff = ManagedFileAccess.file(f); 470 if (ff.exists()) { 471 FileUtilities.atomicDeleteDirectory(f); 472 } 473 return null; 474 }, lockParameters); 475 } 476 477 /** 478 * Load the identified package from the cache - if it exists 479 * <p/> 480 * This is for special purpose only (testing, control over speed of loading). 481 * <p/> 482 * Generally, use the loadPackage method 483 * 484 * @param id The id of the package to load 485 * @param version The version of the package to load. Values such as 'current' and 'dev' are allowed. 486 * @return The package, or null if it is not found 487 * @throws IOException If the package cannot be loaded 488 */ 489 @Override 490 public NpmPackage loadPackageFromCacheOnly(String id, @Nullable String version) throws IOException { 491 id = stripAlias(id); 492 493 if (!Utilities.noString(version) && version.startsWith("file:")) { 494 return loadPackageFromFile(id, version.substring(5)); 495 } 496 497 for (NpmPackage p : temporaryPackages) { 498 if (p.name().equals(id) && ("current".equals(version) || "dev".equals(version) || p.version().equals(version))) { 499 return p; 500 } 501 if (p.name().equals(id) && Utilities.noString(version)) { 502 return p; 503 } 504 } 505 506 String foundPackageFolder = findPackageFolder(id, version); 507 if (foundPackageFolder != null) { 508 NpmPackage foundPackage = locks.getPackageLock(foundPackageFolder).doReadWithLock(() -> { 509 String path = Utilities.path(cacheFolder, foundPackageFolder); 510 File directory = ManagedFileAccess.file(path); 511 512 /* Check if the directory still exists now that we have a read lock. findPackageFolder does no locking in order 513 to avoid locking every potential package directory, so it's possible that a package deletion has occurred. 514 * */ 515 if (!directory.exists()) { 516 return null; 517 } 518 return loadPackageInfo(path); 519 }, lockParameters); 520 if (foundPackage != null) { 521 if (foundPackage.isIndexed()){ 522 return foundPackage; 523 } else { 524 return locks.getPackageLock(foundPackageFolder).doWriteWithLock(() -> { 525 File directory = ManagedFileAccess.file(foundPackage.getPath()); 526 527 /* Check if the directory still exists now that we have a write lock. findPackageFolder does no locking in order 528 to avoid locking every potential package directory, so it's possible that a package deletion has occurred. 529 * */ 530 if (!directory.exists()) { 531 return null; 532 } 533 534 // Since another thread may have already indexed the package since our read, we need to check again 535 NpmPackage output = loadPackageInfo(foundPackage.getPath()); 536 if (output.isIndexed()) { 537 return output; 538 } 539 String path = Utilities.path(cacheFolder, foundPackageFolder); 540 output.checkIndexed(path); 541 return output; 542 }, lockParameters); 543 } 544 } 545 } 546 if ("dev".equals(version)) 547 return loadPackageFromCacheOnly(id, "current"); 548 else 549 return null; 550 } 551 552 private String findPackageFolder(String id, String version) throws IOException { 553 String foundPackageFolder = null; 554 String foundVersion = null; 555 for (String currentPackageFolder : Utilities.reverseSorted(cacheFolder.list())) { 556 File cf = ManagedFileAccess.file(Utilities.path(cacheFolder, currentPackageFolder)); 557 if (cf.isDirectory()) { 558 if (currentPackageFolder.equals(id + "#" + version) || (Utilities.noString(version) && currentPackageFolder.startsWith(id + "#"))) { 559 return currentPackageFolder; 560 } 561 if (version != null && !Utilities.existsInList(version, "current", "dev") && (VersionUtilities.isSemVerWithWildcards(version) || Utilities.charCount(version, '.') < 2) && currentPackageFolder.contains("#")) { 562 String[] parts = currentPackageFolder.split("#"); 563 if (parts[0].equals(id) && VersionUtilities.isSemVer(parts[1]) && VersionUtilities.versionMatches((foundVersion != null ? foundVersion : version), parts[1])) { 564 foundVersion = parts[1]; 565 foundPackageFolder = currentPackageFolder; 566 } 567 } 568 } 569 } 570 return foundPackageFolder; 571 } 572 573 /** 574 * Add an already fetched package to the cache 575 */ 576 @Override 577 public NpmPackage addPackageToCache(String id, final String version, final InputStream packageTgzInputStream, final String sourceDesc) throws IOException { 578 String sid = stripAlias(id); 579 checkValidVersionString(version, sid); 580 return locks.getPackageLock(sid + "#" + version).doWriteWithLock(() -> { 581 582 String tempDir = Utilities.generateUniqueRandomUUIDPath(cacheFolder.getAbsolutePath()); 583 584 NpmPackage extractedNpm = NpmPackage.extractFromTgz(packageTgzInputStream, sourceDesc, tempDir, minimalMemory); 585 586 log.info(""); 587 log.info("Installing " + sid + "#" + version); 588 589 if ((extractedNpm.name() != null && sid != null && !sid.equalsIgnoreCase(extractedNpm.name()) && !sid.equalsIgnoreCase(extractedNpm.name()+"."+VersionUtilities.getNameForVersion(extractedNpm.fhirVersion())))) { 590 if (!suppressErrors && (!sid.equals("hl7.fhir.r5.core") && !sid.equals("hl7.fhir.us.immds"))) {// temporary work around 591 throw new IOException("Attempt to import a mis-identified package. Expected " + sid + ", got " + extractedNpm.name()); 592 } 593 } 594 595 NpmPackage npmPackage; 596 String packageRoot = Utilities.path(cacheFolder, sid + "#" + version); 597 try { 598 if (!sid.equals(extractedNpm.getNpm().asString("name")) || !version.equals(extractedNpm.getNpm().asString("version"))) { 599 if (!sid.equals(extractedNpm.getNpm().asString("name"))) { 600 extractedNpm.getNpm().add("original-name", extractedNpm.getNpm().asString("name")); 601 extractedNpm.getNpm().remove("name"); 602 extractedNpm.getNpm().add("name", sid); 603 } 604 if (!version.equals(extractedNpm.getNpm().asString("version"))) { 605 extractedNpm.getNpm().add("original-version", extractedNpm.getNpm().asString("version")); 606 extractedNpm.getNpm().remove("version"); 607 extractedNpm.getNpm().add("version", version); 608 } 609 FileUtilities.stringToFile(JsonParser.compose(extractedNpm.getNpm(), true), Utilities.path(tempDir, "package", "package.json")); 610 } 611 612 final NpmPackage tempPackage = loadPackageInfo(tempDir); 613 if (tempPackage != null && !tempPackage.isIndexed()) { 614 tempPackage.checkIndexed(packageRoot); 615 } 616 617 if (!ManagedFileAccess.file(packageRoot).exists() || Utilities.existsInList(version, "current", "dev")) { 618 FileUtilities.createDirectory(packageRoot); 619 try { 620 FileUtilities.clearDirectory(packageRoot); 621 } catch (Throwable t) { 622 log.info("Unable to clear directory: " + packageRoot + ": " + t.getMessage() + " - this may cause problems later"); 623 } 624 FileUtilities.renameDirectory(tempDir, packageRoot); 625 626 npmPackage = loadPackageInfo(packageRoot); 627 628 log.info(" done."); 629 } else { 630 FileUtilities.clearDirectory(tempDir); 631 ManagedFileAccess.file(tempDir).delete(); 632 } 633 npmPackage = loadPackageInfo(packageRoot); 634 635 } catch (Exception e) { 636 try { 637 // don't leave a half extracted package behind 638 log.info("Clean up package " + packageRoot + " because installation failed: " + e.getMessage()); 639 e.printStackTrace(); 640 FileUtilities.clearDirectory(packageRoot); 641 ManagedFileAccess.file(packageRoot).delete(); 642 } catch (Exception ignored) { 643 // nothing 644 } 645 throw e; 646 } 647 return npmPackage; 648 }, lockParameters); 649 } 650 651 @Override 652 public String getPackageUrl(String packageId) throws IOException { 653 packageId = stripAlias(packageId); 654 String result = super.getPackageUrl(packageId); 655 if (result == null) { 656 result = ciBuildClient.getPackageUrl(packageId); 657 } 658 return result; 659 } 660 661 /** 662 * do not use this in minimal memory mode 663 * @param packagesFolder 664 * @throws IOException 665 */ 666 public void loadFromFolder(String packagesFolder) throws IOException { 667 assert !minimalMemory; 668 669 File[] files = ManagedFileAccess.file(packagesFolder).listFiles(); 670 if (files != null) { 671 for (File f : files) { 672 if (f.getName().endsWith(".tgz")) { 673 FileInputStream fs = ManagedFileAccess.inStream(f); 674 try { 675 temporaryPackages.add(NpmPackage.fromPackage(fs)); 676 } finally { 677 fs.close(); 678 } 679 } 680 } 681 } 682 } 683 684 @Override 685 public NpmPackage loadPackage(String id, String version) throws FHIRException, IOException { 686 id = stripAlias(id); 687 //ok, try to resolve locally 688 if (!Utilities.noString(version) && version.startsWith("file:")) { 689 return loadPackageFromFile(id, version.substring(5)); 690 } 691 692 if (version == null && id.contains("#")) { 693 version = id.substring(id.indexOf("#") + 1); 694 id = id.substring(0, id.indexOf("#")); 695 } 696 697 if (version == null) { 698 try { 699 version = getLatestVersion(id, false); 700 } catch (Exception e) { 701 version = null; 702 } 703 } 704 NpmPackage p = loadPackageFromCacheOnly(id, version); 705 if (p != null) { 706 if ("current".equals(version)) { 707 p = nullIfNotCurrentPackage(id, p); 708 } 709 if (p != null) 710 return p; 711 } 712 713 if ("dev".equals(version)) { 714 p = loadPackageFromCacheOnly(id, "current"); 715 p = nullIfNotCurrentPackage(id, p); 716 if (p != null) 717 return p; 718 version = "current"; 719 } 720 721 log.info("Installing " + id + "#" + (version == null ? "?" : version) + " to the package cache"); 722 log.info(" Fetching:"); 723 724 // nup, don't have it locally (or it's expired) 725 FilesystemPackageCacheManager.InputStreamWithSrc source; 726 if (false && packageProvider != null && packageProvider.handlesPackage(id, version)) { 727 source = packageProvider.provide(id, version); 728 } else if (Utilities.isAbsoluteUrl(version)) { 729 source = fetchSourceFromUrlSpecific(version); 730 } else if ("current".equals(version) || (version != null && version.startsWith("current$"))) { 731 // special case - fetch from ci-build server 732 source = ciBuildClient.loadFromCIBuild(id, version.startsWith("current$") ? version.substring(8) : null); 733 } else { 734 source = loadFromPackageServer(id, version); 735 } 736 if (source == null) { 737 throw new FHIRException("Unable to find package " + id + "#" + version); 738 } 739 return addPackageToCache(id, source.version, source.stream, source.url); 740 } 741 742 private InputStreamWithSrc fetchSourceFromUrlSpecific(String url) { 743 return new InputStreamWithSrc(fetchFromUrlSpecific(url, false), url, "current"); 744 } 745 746 private InputStream fetchFromUrlSpecific(String source, boolean optional) throws FHIRException { 747 try { 748 HTTPResult res = ManagedWebAccess.get(Arrays.asList("web"), source); 749 res.checkThrowException(); 750 return new ByteArrayInputStream(res.getContent()); 751 } catch (Exception e) { 752 if (optional) 753 return null; 754 else 755 throw new FHIRException("Unable to fetch: " + e.getMessage(), e); 756 } 757 } 758 759 760 761 @Override 762 public String getPackageId(String canonicalUrl) throws IOException { 763 String retVal = findCanonicalInLocalCache(canonicalUrl); 764 765 if (retVal == null) { 766 retVal = super.getPackageId(canonicalUrl); 767 } 768 769 if (retVal == null) { 770 retVal = ciBuildClient.getPackageId(canonicalUrl); 771 } 772 773 return retVal; 774 } 775 776 777 public String findCanonicalInLocalCache(String canonicalUrl) { 778 try { 779 for (String pf : listPackages()) { 780 if (ManagedFileAccess.file(Utilities.path(cacheFolder, pf, "package", "package.json")).exists()) { 781 JsonObject npm = JsonParser.parseObjectFromFile(Utilities.path(cacheFolder, pf, "package", "package.json")); 782 if (canonicalUrl.equals(npm.asString("canonical"))) { 783 return npm.asString("name"); 784 } 785 } 786 } 787 } catch (IOException e) { 788 } 789 return null; 790 } 791 792 // ========================= Package Mgmt API ======================================================================= 793 794 /** 795 * Checks https:// 796 * 797 * @param id 798 * @param npmPackage 799 * @return 800 */ 801 private NpmPackage nullIfNotCurrentPackage(String id, NpmPackage npmPackage) { 802 // special case: current versions roll over, and we have to check their currency 803 try { 804 return ciBuildClient.isCurrent(id, npmPackage) ? npmPackage : null; 805 } catch (Exception e) { 806 log.info("Unable to check package currency: " + id + ": " + id); 807 } 808 return npmPackage; 809 } 810 811 // ----- the old way, from before package server, while everything gets onto the package server 812 private InputStreamWithSrc fetchTheOldWay(String id, String v) { 813 String url = getUrlForPackage(id); 814 if (url == null) { 815 try { 816 url = ciBuildClient.getPackageUrl(id); 817 } catch (Exception ignored) { 818 819 } 820 } 821 if (url == null) { 822 throw new FHIRException("Unable to resolve package id " + id + "#" + v); 823 } 824 if (url.contains("/ImplementationGuide/")) { 825 url = url.substring(0, url.indexOf("/ImplementationGuide/")); 826 } 827 String pu = Utilities.pathURL(url, "package-list.json"); 828 829 PackageList pl; 830 try { 831 pl = PackageList.fromUrl(pu); 832 } catch (Exception e) { 833 String pv = Utilities.pathURL(url, v, "package.tgz"); 834 try { 835 return new InputStreamWithSrc(fetchFromUrlSpecific(pv, false), pv, v); 836 } catch (Exception e1) { 837 throw new FHIRException("Error fetching package directly (" + pv + "), or fetching package list for " + id + " from " + pu + ": " + e1.getMessage(), e1); 838 } 839 } 840 if (!id.equals(pl.pid())) 841 throw new FHIRException("Package ids do not match in " + pu + ": " + id + " vs " + pl.pid()); 842 for (PackageListEntry vo : pl.versions()) { 843 if (v.equals(vo.version())) { 844 845 String u = Utilities.pathURL(vo.path(), "package.tgz"); 846 return new InputStreamWithSrc(fetchFromUrlSpecific(u, true), u, v); 847 } 848 } 849 850 return null; 851 } 852 853 854 // ---------- Current Build SubSystem -------------------------------------------------------------------------------------- 855 856 private String fetchVersionTheOldWay(String id) throws IOException { 857 String url = getUrlForPackage(id); 858 if (url == null) { 859 try { 860 url = ciBuildClient.getPackageUrl(id); 861 } catch (Exception e) { 862 url = null; 863 } 864 } 865 if (url == null) { 866 throw new FHIRException("Unable to resolve package id " + id); 867 } 868 PackageList pl = PackageList.fromUrl(Utilities.pathURL(url, "package-list.json")); 869 if (!id.equals(pl.pid())) 870 throw new FHIRException("Package ids do not match in " + pl.source() + ": " + id + " vs " + pl.pid()); 871 for (PackageListEntry vo : pl.versions()) { 872 if (vo.current()) { 873 return vo.version(); 874 } 875 } 876 877 return null; 878 } 879 880 private String fetchVersionTheOldWay(String id, String versionSpec) throws IOException { 881 String url = getUrlForPackage(id); 882 if (url == null) { 883 try { 884 url = ciBuildClient.getPackageUrl(id); 885 } catch (Exception e) { 886 url = null; 887 } 888 } 889 if (url == null) { 890 throw new FHIRException("Unable to resolve package id " + id); 891 } 892 PackageList pl = PackageList.fromUrl(Utilities.pathURL(url, "package-list.json")); 893 if (!id.equals(pl.pid())) 894 throw new FHIRException("Package ids do not match in " + pl.source() + ": " + id + " vs " + pl.pid()); 895 for (PackageListEntry vo : pl.versions()) { 896 if (VersionUtilities.versionMatches(versionSpec, vo.version())) { 897 return vo.version(); 898 } 899 } 900 901 return null; 902 } 903 904 private String getUrlForPackage(String id) { 905 if (CommonPackages.ID_XVER.equals(id)) { 906 return "https://fhir.org/packages/hl7.fhir.xver-extensions"; 907 } 908 return null; 909 } 910 911 public List<String> listPackages() { 912 List<String> res = new ArrayList<>(); 913 for (File f : cacheFolder.listFiles()) { 914 if (f.isDirectory() && f.getName().contains("#")) { 915 res.add(f.getName()); 916 } 917 } 918 return res; 919 } 920 921 public interface CacheLockFunction<T> { 922 T get() throws IOException; 923 } 924 925 926 927 928 929 public boolean packageExists(String id, String ver) throws IOException { 930 id = stripAlias(id); 931 if (packageInstalled(id, ver)) { 932 return true; 933 } 934 for (PackageServer s : getPackageServers()) { 935 if (new PackageClient(s).exists(id, ver)) { 936 return true; 937 } 938 } 939 return false; 940 } 941 942 public boolean packageInstalled(String id, String version) { 943 id = stripAlias(id); 944 for (NpmPackage p : temporaryPackages) { 945 if (p.name().equals(id) && ("current".equals(version) || "dev".equals(version) || p.version().equals(version))) { 946 return true; 947 } 948 if (p.name().equals(id) && Utilities.noString(version)) { 949 return true; 950 } 951 } 952 953 for (String f : Utilities.sorted(cacheFolder.list())) { 954 if (f.equals(id + "#" + version) || (Utilities.noString(version) && f.startsWith(id + "#"))) { 955 return true; 956 } 957 } 958 if ("dev".equals(version)) 959 return packageInstalled(id, "current"); 960 else 961 return false; 962 } 963 964 public boolean isSuppressErrors() { 965 return suppressErrors; 966 } 967 968 public void setSuppressErrors(boolean suppressErrors) { 969 this.suppressErrors = suppressErrors; 970 } 971 972 public static IPackageProvider getPackageProvider() { 973 return packageProvider; 974 } 975 976 public static void setPackageProvider(IPackageProvider packageProvider) { 977 FilesystemPackageCacheManager.packageProvider = packageProvider; 978 } 979 980 981}