001package org.hl7.fhir.utilities.npm;
002
003/*
004  Copyright (c) 2011+, HL7, Inc.
005  All rights reserved.
006  
007  Redistribution and use in source and binary forms, with or without modification, 
008  are permitted provided that the following conditions are met:
009    
010   * Redistributions of source code must retain the above copyright notice, this 
011     list of conditions and the following disclaimer.
012   * Redistributions in binary form must reproduce the above copyright notice, 
013     this list of conditions and the following disclaimer in the documentation 
014     and/or other materials provided with the distribution.
015   * Neither the name of HL7 nor the names of its contributors may be used to 
016     endorse or promote products derived from this software without specific 
017     prior written permission.
018  
019  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
020  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
021  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 
022  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 
023  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 
024  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
025  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
026  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
027  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
028  POSSIBILITY OF SUCH DAMAGE.
029  
030 */
031
032
033
034import java.io.BufferedOutputStream;
035import java.io.ByteArrayInputStream;
036import java.io.ByteArrayOutputStream;
037import java.io.File;
038import java.io.FileNotFoundException;
039import java.io.FileOutputStream;
040import java.io.IOException;
041import java.io.InputStream;
042import java.io.OutputStream;
043import java.nio.charset.StandardCharsets;
044import java.text.ParseException;
045import java.text.SimpleDateFormat;
046import java.time.LocalDate;
047import java.util.ArrayList;
048import java.util.Arrays;
049import java.util.Collection;
050import java.util.Collections;
051import java.util.Comparator;
052import java.util.Date;
053import java.util.HashMap;
054import java.util.List;
055import java.util.Map;
056import java.util.Set;
057import java.util.UUID;
058import java.util.zip.Deflater;
059import java.util.zip.ZipEntry;
060import java.util.zip.ZipInputStream;
061
062import javax.annotation.Nonnull;
063
064import lombok.Getter;
065import lombok.extern.slf4j.Slf4j;
066import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
067import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
068import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
069import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
070import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
071import org.apache.commons.compress.compressors.gzip.GzipParameters;
072import org.apache.commons.lang3.Validate;
073import org.hl7.fhir.exceptions.FHIRException;
074import org.hl7.fhir.utilities.ByteProvider;
075import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
076import org.hl7.fhir.utilities.FileUtilities;
077import org.hl7.fhir.utilities.Utilities;
078import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
079import org.hl7.fhir.utilities.http.HTTPResult;
080import org.hl7.fhir.utilities.http.ManagedWebAccess;
081import org.hl7.fhir.utilities.json.JsonException;
082import org.hl7.fhir.utilities.json.model.JsonArray;
083import org.hl7.fhir.utilities.json.model.JsonElement;
084import org.hl7.fhir.utilities.json.model.JsonObject;
085import org.hl7.fhir.utilities.json.model.JsonProperty;
086import org.hl7.fhir.utilities.json.parser.JsonParser;
087import org.hl7.fhir.utilities.npm.PackageGenerator.PackageType;
088
089/**
090 * info and loader for a package 
091 * 
092 * Packages may exist on disk in the cache, or purely in memory when they are loaded on the fly
093 * 
094 * Packages are contained in subfolders (see the package spec). The FHIR resources will be in "package"
095 * 
096 * @author Grahame Grieve
097 *
098 */
099@Slf4j
100public class NpmPackage {
101
102  public interface ITransformingLoader {
103
104    byte[] load(File f);
105
106  }
107
108  public class PackageResourceInformationSorter implements Comparator<PackageResourceInformation> {
109    @Override
110    public int compare(PackageResourceInformation o1, PackageResourceInformation o2) {
111      return o1.filename.compareTo(o2.filename);
112    }
113  }
114  
115  public class PackageResourceInformation {
116    private final String id;
117    private final String resourceType;
118    private final String url;
119    private final String version;
120    private final String filename;
121    private final String supplements;
122    private final String stype;
123    private final String derivation;
124    private final String content;
125    
126    public PackageResourceInformation(String root, JsonObject fi) throws IOException {
127      super();
128      id = fi.asString("id");
129      resourceType = fi.asString("resourceType");
130      url = fi.asString("url");
131      version = fi.asString("version");
132      filename = Utilities.path(root, fi.asString("filename"));
133      supplements = fi.asString("supplements");
134      stype = fi.asString("type");
135      derivation = fi.asString("derivation");
136      content = fi.asString("content");
137    }
138    public String getId() {
139      return id;
140    }
141    public String getResourceType() {
142      return resourceType;
143    }
144    public String getStatedType() {
145      return stype;
146    }
147    public String getUrl() {
148      return url;
149    }
150    public String getVersion() {
151      return version;
152    }
153    public String getFilename() {
154      return filename;
155    }
156    public String getSupplements() {
157      return supplements;
158    }
159    public boolean hasId() {
160      return !Utilities.noString(id);
161    }
162    public String getDerivation() {
163      return derivation;
164    }
165    public String getContent() {
166      return content;
167    }
168    
169  }
170  public class IndexVersionSorter implements Comparator<JsonObject> {
171
172    @Override
173    public int compare(JsonObject o0, JsonObject o1) {
174      String v0 = o0.asString("version"); 
175      String v1 = o1.asString("version"); 
176      return v0.compareTo(v1);
177    }
178  }
179
180  public static class PackagedResourceFile {
181    @Getter
182    private final String folder;
183    @Getter
184    private final String filename;
185    @Getter
186    private final String resourceType;
187    @Getter
188    private final boolean example;
189    protected PackagedResourceFile(String folder, String filename, String resourceType, boolean example) {
190      super();
191      this.folder = folder;
192      this.filename = filename;
193      this.resourceType = resourceType;
194      this.example = example;
195    }
196
197    public static class Sorter implements Comparator<PackagedResourceFile> {
198
199      @Override
200      public int compare(PackagedResourceFile o1, PackagedResourceFile o2) {
201        int res = o1.folder.compareTo(o2.folder);
202        if (res == 0) {
203          res = o1.filename.compareTo(o2.filename);
204        }
205        return res;
206      }
207      
208    }
209  }
210  
211  public static boolean isValidName(String pid) {
212    return pid.matches("^[a-z][a-zA-Z0-9]*(\\.[a-z][a-zA-Z0-9\\-]*)+$");
213  }
214
215  public static boolean isValidVersion(String ver) {
216    return ver.matches("^[0-9]+\\.[0-9]+\\.[0-9]+$");
217  }
218
219  public class NpmPackageFolder {
220    private final String folderName;
221    private Map<String, List<String>> types;
222    private Map<String, byte[]> content;
223    private JsonObject cachedIndex;
224    private File folder;
225
226    public NpmPackageFolder(String folderName) {
227      super();
228      this.folderName = folderName;
229      if (!minimalMemory) {
230        types = new HashMap<>();
231        content = new HashMap<>();
232      }
233    }
234
235    private String fn(String name) throws IOException {
236      return Utilities.path(folder.getAbsolutePath(), name);
237    }
238
239    public Map<String, List<String>> getTypes() throws JsonException, IOException {
240      if (minimalMemory) {
241        Map<String, List<String>> typeMap = new HashMap<>();
242        readIndex(JsonParser.parseObjectFromFile(fn(".index.json")), typeMap);
243        return typeMap;
244      } else {
245        return types;
246      }
247    }
248
249    public String getFolderName() {
250      return folderName;
251    }
252
253    public String getFolderPath() {
254      return folder == null ? null : folder.getAbsolutePath();
255    }
256    
257    public boolean readIndex(JsonObject index, Map<String, List<String>> typeMap) {
258      if (!index.has("index-version") || (index.asInteger("index-version") != NpmPackageIndexBuilder.CURRENT_INDEX_VERSION)) {
259        return false;
260      }
261      if (!minimalMemory) {
262        this.cachedIndex = index;
263      }
264      for (JsonObject file : index.getJsonObjects("files")) {
265        String type = file.asString("resourceType");
266        String name = file.asString("filename");
267        if (!typeMap.containsKey(type))
268          typeMap.put(type, new ArrayList<>());
269        typeMap.get(type).add(name);
270      }
271      return true;
272    }
273
274    public List<String> listFiles() {
275      List<String> res = new ArrayList<>();
276      if (folder != null) {
277        if (folder.exists()) {
278          for (File f : folder.listFiles()) {
279            if (!f.isDirectory() && !Utilities.existsInList(f.getName(), "package.json", ".index.json", ".index.db", ".oids.json", ".oids.db")) {
280              res.add(f.getName());
281            }
282          }
283        }
284      } else {
285        for (String s : content.keySet()) {
286          if (!Utilities.existsInList(s, "package.json", ".index.json", ".index.db", ".oids.json", ".oids.db")) {
287            res.add(s);
288          }
289        }
290      }
291      Collections.sort(res);
292      return res;
293    }
294
295    public Map<String, byte[]> getContent() {
296      assert !minimalMemory;
297      
298      return content;
299    }
300
301    public byte[] fetchFile(String file) throws IOException {
302      if (folder != null) {
303        File f = ManagedFileAccess.file(Utilities.path(folder.getAbsolutePath(), file));
304        if (f.exists()) {
305          return FileUtilities.fileToBytes(f);
306        } else {
307          return null;
308        }
309      } else {
310        return content.get(file);
311      }
312    }
313
314    public ByteProvider getProvider(String file) throws IOException {
315      if (folder != null) {
316        File f = ManagedFileAccess.file(Utilities.path(folder.getAbsolutePath(), file));
317        if (f.exists()) {
318          return ByteProvider.forFile(f);
319        } else {
320          return null;
321        }
322      } else {
323        return ByteProvider.forBytes(content.get(file));
324      }
325    }
326
327    public boolean hasFile(String file) throws IOException {
328      if (folder != null) {
329        return ManagedFileAccess.file(Utilities.path(folder.getAbsolutePath(), file)).exists();
330      } else {
331        return content.containsKey(file);
332      }
333
334    }
335
336    public String dump() {
337      return folderName + " ("+ (folder == null ? "null" : folder.toString())+")"+(minimalMemory ? "" : " | " + (cachedIndex != null) + " | " + content.size() + " | " + types.size());
338    }
339
340    public void removeFile(String n) throws IOException {
341      if (folder != null) {
342        ManagedFileAccess.file(Utilities.path(folder.getAbsolutePath(), n)).delete();
343      } else {
344        content.remove(n);
345      }
346      changedByLoader = true;      
347    }
348
349    public JsonObject index() throws IOException {
350      if (cachedIndex != null) {
351        return cachedIndex;
352      } else if (folder == null) {
353        return null;
354      } else {
355        File ij = ManagedFileAccess.file(fn(".index.json"));
356        if (ij.exists()) {
357          return JsonParser.parseObject(ij);
358        } else {
359          return null;
360        }
361      }
362    }
363    public JsonObject oidIndex() throws IOException {
364      if (folder == null) {
365        return null;
366      } else {
367        File ij = ManagedFileAccess.file(fn(".oids.json"));
368        if (ij.exists()) {
369          return JsonParser.parseObject(ij);
370        } else {
371          return null;
372        }
373      }
374    }
375
376
377  }
378
379
380  private String path;
381  private JsonObject npm;
382  private final Map<String, NpmPackageFolder> folders = new HashMap<>();
383  private boolean changedByLoader; // internal qa only!
384  private Map<String, Object> userData;
385  private boolean minimalMemory;
386  private int size;
387  private boolean warned = false;
388  private static boolean loadCustomResources;
389
390  /**
391   * Constructor
392   */
393  private NpmPackage() {
394    super();
395  }
396
397  /**
398   * Factory method that parses a package from an extracted folder
399   */
400  public static NpmPackage fromFolder(String path) throws IOException {
401    return fromFolder(path, true);
402  }
403
404  public static NpmPackage fromFolder(String path, boolean checkIndexed) throws IOException {
405    NpmPackage res = new NpmPackage();
406    res.loadFiles(path, ManagedFileAccess.file(path));
407    if (checkIndexed) {
408     res.checkIndexed(path);
409    }
410    return res;
411  }
412
413  /**
414   * Factory method that parses a package from an extracted folder
415   */
416  public static NpmPackage fromFolderMinimal(String path) throws IOException {
417    return fromFolderMinimal(path, true);
418  }
419
420  public static NpmPackage fromFolderMinimal(String path, boolean checkIndexed) throws IOException {
421    NpmPackage res = new NpmPackage();
422    res.minimalMemory = true;
423    res.loadFiles(path, ManagedFileAccess.file(path));
424    if (checkIndexed) {
425    res.checkIndexed(path);}
426    return res;
427  }
428
429  /**
430   * Factory method that starts a new empty package using the given PackageGenerator to create the manifest
431   */
432  public static NpmPackage empty(PackageGenerator thePackageGenerator) {
433    NpmPackage retVal = new NpmPackage();
434    retVal.npm = PackageHacker.fixPackageOnLoad(thePackageGenerator.getRootJsonObject());
435    return retVal;
436  }
437
438  /**
439   * Factory method that starts a new empty package using the given PackageGenerator to create the manifest
440   */
441  public static NpmPackage empty() {
442    NpmPackage retVal = new NpmPackage();
443    return retVal;
444  }
445
446  public Map<String, Object> getUserData() {
447    if (userData == null) {
448      userData = new HashMap<>();
449    }
450    return userData;
451  }
452
453  public void loadFiles(String path, File source, String... exemptions) throws IOException {
454    this.npm = PackageHacker.fixPackageOnLoad(JsonParser.parseObject(FileUtilities.fileToString(Utilities.path(path, "package", "package.json"))));
455    this.path = path;
456
457    File dir = ManagedFileAccess.file(path);
458    for (File f : dir.listFiles()) {
459      if (!isInternalExemptFile(f) && !Utilities.existsInList(f.getName(), exemptions)) {
460        if (f.isDirectory()) {
461          String d = f.getName();
462          if (!d.equals("package")) {
463            d = Utilities.path("package", d);
464          }
465          File ij = ManagedFileAccess.file(Utilities.path(f.getAbsolutePath(), ".index.json"));
466          NpmPackageFolder folder = this.new NpmPackageFolder(d);
467          folder.folder = f;
468          this.folders.put(d, folder);
469          if (ij.exists() || !minimalMemory) {
470            if (!minimalMemory) {
471              try {
472                if (!ij.exists() || !folder.readIndex(JsonParser.parseObject(ij), folder.getTypes())) {
473                  indexFolder(folder.getFolderName(), folder);
474                }
475              } catch (Exception e) {
476                throw new IOException("Error parsing "+ij.getAbsolutePath()+": "+e.getMessage(), e);
477              }
478            }
479          }
480          loadSubFolders(dir.getAbsolutePath(), f);
481
482        } else {
483          NpmPackageFolder folder = this.new NpmPackageFolder(Utilities.path("package", "$root"));
484          folder.folder = dir;
485          this.folders.put(Utilities.path("package", "$root"), folder);        
486        }
487      }
488    }
489  }
490
491  public static boolean isInternalExemptFile(File f) {
492    return Utilities.existsInList(f.getName(), ".git", ".svn", ".DS_Store") || Utilities.existsInList(f.getName(), "package-list.json") ||
493        Utilities.endsWithInList(f.getName(), ".tgz");
494  }
495
496  private void loadSubFolders(String rootPath, File dir) throws IOException {
497    for (File f : dir.listFiles()) {
498      if (f.isDirectory()) {
499        if (!"custom".equals(f.getName()) || loadCustomResources) {
500          String d = f.getAbsolutePath().substring(rootPath.length()+1);
501          if (!d.startsWith("package")) {
502            d = Utilities.path("package", d);
503          }
504          NpmPackageFolder folder = this.new NpmPackageFolder(d);
505          folder.folder = f;
506          this.folders.put(d, folder);
507          File ij = ManagedFileAccess.file(Utilities.path(f.getAbsolutePath(), ".index.json"));
508          if (ij.exists() || !minimalMemory) {
509            try {
510              if (!ij.exists() || !folder.readIndex(JsonParser.parseObject(ij), folder.getTypes())) {
511                indexFolder(folder.getFolderName(), folder);
512              }
513            } catch (Exception e) {
514              throw new IOException("Error parsing "+ij.getAbsolutePath()+": "+e.getMessage(), e);
515            }
516          }
517          loadSubFolders(rootPath, f); 
518        }
519      }
520    }    
521  }
522
523  public static NpmPackage fromFolder(String folder, PackageType defType, String... exemptions) throws IOException {
524    NpmPackage res = new NpmPackage();
525    res.loadFiles(folder, ManagedFileAccess.file(folder), exemptions);
526    if (!res.folders.containsKey("package")) {
527      res.folders.put("package", res.new NpmPackageFolder("package"));
528    }
529    if (!res.folders.get("package").hasFile("package.json") && defType != null) {
530      FileUtilities.stringToFile("{ \"type\" : \""+defType.getCode()+"\"}", Utilities.path(res.folders.get("package").folder.getAbsolutePath(), "package.json"));
531    }
532    res.npm = PackageHacker.fixPackageOnLoad(JsonParser.parseObject(new String(res.folders.get("package").fetchFile("package.json"))));
533    return res;
534  }
535
536  private static final int BUFFER_SIZE = 1024;
537
538  public static @Nonnull NpmPackage fromPackage(InputStream tgz) throws IOException {
539    return fromPackage(tgz, null, false);
540  }
541
542  public static NpmPackage fromPackage(InputStream tgz, String desc) throws IOException {
543    return fromPackage(tgz, desc, false);
544  }
545
546  public static NpmPackage fromPackage(InputStream tgz, String desc, boolean progress) throws IOException {
547    NpmPackage res = new NpmPackage();
548    res.readStream(tgz, desc, progress);
549    return res;
550  }
551  
552  public static NpmPackage extractFromTgz(InputStream tgz, String desc, String tempDir, boolean minimal) throws IOException {
553    FileUtilities.createDirectory(tempDir);
554
555    int size = 0;
556    
557    GzipCompressorInputStream gzipIn;
558    try {
559      gzipIn = new GzipCompressorInputStream(tgz);
560    } catch (Exception e) {
561      throw new IOException("Error reading "+(desc == null ? "package" : desc)+": "+e.getMessage(), e);      
562    }
563    try (TarArchiveInputStream tarIn = new TarArchiveInputStream(gzipIn)) {
564      TarArchiveEntry entry;
565
566      while ((entry = tarIn.getNextEntry()) != null) {
567        String n = entry.getName();
568        if (n.contains("/..") || n.contains("../")) {
569          throw new RuntimeException("Entry with an illegal name: " + n);
570        }
571        if (entry.isDirectory()) {
572          if (!Utilities.noString(n)) {
573            String dir = n.substring(0, n.length()-1);
574            FileUtilities.createDirectory(Utilities.path(tempDir, dir));
575          }
576        } else {
577          int count;
578          byte[] data = new byte[BUFFER_SIZE];
579          String filename = Utilities.path(tempDir, n);
580          String folder = FileUtilities.getDirectoryForFile(filename);
581          FileUtilities.createDirectory(folder);
582          FileOutputStream fos = ManagedFileAccess.outStream(filename);
583          try (BufferedOutputStream dst = new BufferedOutputStream(fos, BUFFER_SIZE)) {
584            while ((count = tarIn.read(data, 0, BUFFER_SIZE)) != -1) {
585              dst.write(data, 0, count);
586              size = size + count;
587            }
588          }
589          fos.close();
590        }
591      }
592    } 
593    try {
594      NpmPackage npm = NpmPackage.fromFolderMinimal(tempDir);
595      npm.setSize(size);
596      if (!minimal) {
597        npm.checkIndexed(desc);
598      }
599      return npm;
600    } catch (Exception e) {
601      throw new IOException("Error parsing "+(desc == null ? "" : desc+"#")+"package/package.json: "+e.getMessage(), e);
602    } 
603  }
604  
605  public void readStream(InputStream tgz, String desc, boolean progress) throws IOException {
606    GzipCompressorInputStream gzipIn;
607    try {
608      gzipIn = new GzipCompressorInputStream(tgz);
609    } catch (Exception e) {
610      throw new IOException("Error reading "+(desc == null ? "package" : desc)+": "+e.getMessage(), e);      
611    }
612
613    boolean haveLoggedDotSlashPrefixWarning = false;
614
615    try (TarArchiveInputStream tarIn = new TarArchiveInputStream(gzipIn)) {
616      TarArchiveEntry entry;
617
618      NpmPackageReadLogger readLogger = new NpmPackageReadLogger(progress);
619      while ((entry = (TarArchiveEntry) tarIn.getNextEntry()) != null) {
620        String entryName = entry.getName();
621        if (entryName.contains("..")) {
622          throw new RuntimeException("Entry with an illegal name: " + entryName);
623        }
624        if (entry.isDirectory()) {
625          String dir = entryName.substring(0, entryName.length()-1);
626          // If resource paths in the TGZ file are prefixed with "./", be tolerant
627          // and strip that
628          if (dir.startsWith("./")) {
629            if (!haveLoggedDotSlashPrefixWarning) {
630              log.warn("The NPM file contains resource paths that are prefixed with \"./\". This is invalid and should be corrected in the source package.");
631              haveLoggedDotSlashPrefixWarning = true;
632            }
633            dir = dir.substring(2);
634          }
635          if (dir.startsWith("package/")) {
636            dir = dir.substring(8);
637          }
638          folders.put(dir, new NpmPackageFolder(dir));
639        } else {
640          int count;
641          byte[] data = new byte[BUFFER_SIZE];
642          ByteArrayOutputStream fos = new ByteArrayOutputStream();
643          try (BufferedOutputStream dest = new BufferedOutputStream(fos, BUFFER_SIZE)) {
644            while ((count = tarIn.read(data, 0, BUFFER_SIZE)) != -1) {
645              dest.write(data, 0, count);
646            }
647          }
648          fos.close();
649          loadFile(entryName, fos.toByteArray());
650        }
651        readLogger.entry(entryName);
652      }
653    } 
654    NpmPackageFolder packageFolder = folders.get("package");
655    Validate.notNull(packageFolder, "Package folder not found in NPM file");
656    byte[] packageJsonBytes = packageFolder.fetchFile("package.json");
657    Validate.notNull(packageJsonBytes, "package/package.json not found in NPM file");
658    try {
659      npm = PackageHacker.fixPackageOnLoad(JsonParser.parseObject(packageJsonBytes));
660    } catch (Exception e) {
661      throw new IOException("Error parsing "+(desc == null ? "" : desc+"#")+"package/package.json: "+e.getMessage(), e);
662    }
663    checkIndexed(desc);
664  }
665
666  public void loadFile(String n, byte[] data) throws IOException {
667    if (n.startsWith("./")) {
668      n = n.substring(2);
669    }
670    // work around a bug:
671    n = n.replace("//", "/");
672    String dir = n.contains("/") ? n.substring(0, n.lastIndexOf("/")) : "$root";
673    if (dir.startsWith("package/")) {
674      dir = dir.substring(8);
675    }
676    n = n.substring(n.lastIndexOf("/")+1);
677    NpmPackageFolder index = folders.get(dir);
678    if (index == null) {
679      index = new NpmPackageFolder(dir);
680      folders.put(dir, index);
681    }
682    index.content.put(n, data);
683  }
684
685  public boolean isIndexed() throws IOException {
686    for (NpmPackageFolder folder : folders.values()) {
687      JsonObject index = folder.index();
688      if (folder.index() == null || index.forceArray("files").size() == 0) {
689        return false;
690      }
691    }
692    return true;
693  }
694
695
696  public void checkIndexed(String path) throws IOException {
697    for (NpmPackageFolder folder : folders.values()) {
698      JsonObject index = folder.index();
699      if (index == null || index.forceArray("files").size() == 0) {
700        indexFolder(path, folder);
701      }  
702    }
703  }
704
705
706  /**
707   * Create a package .index.json file for a package folder.
708   * <p>
709   * See <a href="https://hl7.org/fhir/packages.html#2.1.10.4">the FHIR specification</a> for details on .index.json
710   * format and usage.
711   *
712   * @param path
713   * @param folder
714   * @throws FileNotFoundException
715   * @throws IOException
716   */
717  public void indexFolder(String path, NpmPackageFolder folder) throws FileNotFoundException, IOException {
718    List<String> remove = new ArrayList<>();
719    NpmPackageIndexBuilder indexer = new NpmPackageIndexBuilder();
720    indexer.start(folder.folder != null ? Utilities.path(folder.folder.getAbsolutePath(), ".index.db") : null);
721    for (String file : folder.listFiles()) {
722      if (!indexer.seeFile(file, folder.fetchFile(file))) {
723        remove.add(file);
724      }
725    } 
726    for (String n : remove) {
727      folder.removeFile(n);
728    }
729    String json = indexer.build();
730    
731    try {
732      if (!minimalMemory) {
733        folder.readIndex(JsonParser.parseObject(json), folder.getTypes());
734      }
735      if (folder.folder != null) {
736        FileUtilities.stringToFile(json, Utilities.path(folder.folder.getAbsolutePath(), ".index.json"));
737      }
738    } catch (Exception e) {
739      FileUtilities.stringToFile(json, Utilities.path("[tmp]", ".index.json"));
740      throw new IOException("Error parsing "+(path == null ? "" : path+"#")+"package/"+folder.folderName+"/.index.json: "+e.getMessage(), e);
741    }
742  }
743
744
745  public static NpmPackage fromZip(InputStream stream, boolean dropRootFolder, String desc) throws IOException {
746    NpmPackage res = new NpmPackage();
747    ZipInputStream zip = new ZipInputStream(stream);
748    ZipEntry ze;
749    while ((ze = zip.getNextEntry()) != null) {
750      int size;
751      byte[] buffer = new byte[2048];
752
753      ByteArrayOutputStream bytes = new ByteArrayOutputStream();
754      BufferedOutputStream bos = new BufferedOutputStream(bytes, buffer.length);
755
756      while ((size = zip.read(buffer, 0, buffer.length)) != -1) {
757        bos.write(buffer, 0, size);
758      }
759      bos.flush();
760      bos.close();
761      if (bytes.size() > 0) {
762        if (dropRootFolder) {
763          res.loadFile(ze.getName().substring(ze.getName().indexOf("/")+1), bytes.toByteArray());
764        } else {
765          res.loadFile(ze.getName(), bytes.toByteArray());
766        }
767      }
768      zip.closeEntry();
769    }
770    zip.close();         
771    try {
772      res.npm = PackageHacker.fixPackageOnLoad(JsonParser.parseObject(res.folders.get("package").fetchFile("package.json")));
773    } catch (Exception e) {
774      throw new IOException("Error parsing "+(desc == null ? "" : desc+"#")+"package/package.json: "+e.getMessage(), e);
775    }
776    res.checkIndexed(desc);
777    return res;
778  }
779
780
781  /**
782   * Accessing the contents of the package - get a list of files in a subfolder of the package 
783   *
784   * @param folder
785   * @return
786   * @throws IOException 
787   */
788  public List<String> list(String folder) throws IOException {
789    List<String> res = new ArrayList<String>();
790    if (folders.containsKey(folder)) {
791      for (String s : folders.get(folder).listFiles()) {
792        if (!s.startsWith(".")) {
793          res.add(s);
794        }
795      }
796    } else if (folders.containsKey(Utilities.path("package", folder))) {
797      for (String s : folders.get(Utilities.path("package", folder)).listFiles()) {
798        if (!s.startsWith(".")) {
799          res.add(s);
800        }
801      }
802    }
803    return res;
804  }
805
806  public List<String> listResources(String... types) throws IOException {
807    return listResources(Utilities.stringSet(types));
808  }
809  
810  public List<String> listResourcesinFolder(String folder, String... types) throws IOException {
811    return listResourcesInFolder(folder, Utilities.stringSet(types));
812  }
813  
814  public List<String> listResources(Set<String> types) throws IOException {
815    return listResourcesInFolder("package", types);
816  }
817  
818  public List<String> listResourcesInFolder(String folderName, Set<String> types) throws IOException {
819    List<String> res = new ArrayList<String>();
820    NpmPackageFolder folder = folders.get(folderName);
821    if (types.size() == 0) {
822      for (String s : folder.types.keySet()) {
823        if (folder.types.containsKey(s)) {
824          res.addAll(folder.types.get(s));
825        }
826      }
827    } else {
828      for (String s : types) {
829        if (folder.types.containsKey(s)) {
830          res.addAll(folder.types.get(s));
831        }
832      }
833    }
834    Collections.sort(res);
835    return res;
836  }
837
838  public List<PackagedResourceFile> listAllResources(Collection<String> types) throws IOException {
839    List<PackagedResourceFile> res = new ArrayList<PackagedResourceFile>();
840    for (NpmPackageFolder folder : folders.values()) {
841      if (!folder.getFolderName().startsWith("tests")) {
842        if (types.size() == 0) {
843          for (String s : folder.types.keySet()) {
844            if (folder.types.containsKey(s)) {
845              for (String n : folder.types.get(s)) {
846                res.add(new PackagedResourceFile(folder.folderName, n, s, folder.getFolderName().equals("example")));
847              }
848            }
849          }
850        } else {
851          for (String s : types) {
852            if (folder.types.containsKey(s)) {
853              for (String n : folder.types.get(s)) {
854                res.add(new PackagedResourceFile(folder.folderName, n, s, folder.getFolderName().equals("example")));
855              }
856            }
857          }
858        }
859      }
860    }
861    Collections.sort(res, new PackagedResourceFile.Sorter());
862    return res;
863  }
864
865  public List<PackagedResourceFile> listAllResources() throws IOException {
866    List<PackagedResourceFile> res = new ArrayList<PackagedResourceFile>();
867    for (NpmPackageFolder folder : folders.values()) {
868      if (!folder.getFolderName().startsWith("tests")) {
869        for (String s : folder.types.keySet()) {
870          if (folder.types.containsKey(s)) {
871            for (String n : folder.types.get(s)) {
872              res.add(new PackagedResourceFile(folder.folderName, n, s, folder.getFolderName().equals("example")));
873            }
874          }
875        }
876      }
877    }
878    Collections.sort(res, new PackagedResourceFile.Sorter());
879    return res;
880  }
881
882  public List<PackageResourceInformation> listIndexedResources(String... types) throws IOException {
883    return listIndexedResources(Utilities.stringSet(types));
884  }
885  
886  public List<PackageResourceInformation> listIndexedResources(Set<String> types) throws IOException {
887    List<PackageResourceInformation> res = new ArrayList<PackageResourceInformation>();
888    for (NpmPackageFolder folder : folders.values()) {
889      JsonObject index = folder.index();
890      if (index != null) {
891        for (JsonObject fi : index.getJsonObjects("files")) {
892          if (types.contains(fi.asString("resourceType")) || types.isEmpty()) {
893            res.add(new PackageResourceInformation(folder.folder == null ? "@"+folder.getFolderName() : folder.folder.getAbsolutePath(), fi));
894          }
895        }
896      }
897    } 
898    //    Collections.sort(res, new PackageResourceInformationSorter());
899    return res;
900  }
901
902  /**
903   * use the name from listResources()
904   * 
905   * @param file
906   * @return
907   * @throws IOException
908   */
909  public InputStream loadResource(String file) throws IOException {
910    NpmPackageFolder folder = folders.get("package");
911    return new ByteArrayInputStream(folder.fetchFile(file));
912  }
913
914  /**
915   * get a stream that contains the contents of a resource in the base folder, by it's canonical URL
916   * 
917   * @param canonical - the canonical URL of the resource (exact match only)
918   * @return null if it is not found
919   * @throws IOException
920   */
921  public InputStream loadByCanonical(String canonical) throws IOException {
922    return loadByCanonicalVersion("package", canonical, null);    
923  }
924  
925  /**
926   * get a stream that contains the contents of a resource in the nominated folder, by it's canonical URL
927   * 
928   * @param folder - one of the folders in the package (main folder is "package")
929   * @param canonical - the canonical URL of the resource (exact match only)
930   * @return null if it is not found
931   * @throws IOException
932   */
933  public InputStream loadByCanonical(String folder, String canonical) throws IOException {
934    return loadByCanonicalVersion(folder, canonical, null);    
935  }
936    
937  /**
938   * get a stream that contains the contents of a resource in the base folder, by it's canonical URL
939   * 
940   * @param canonical - the canonical URL of the resource (exact match only)
941   * @param version - the specified version (or null if the most recent)
942   * 
943   * @return null if it is not found
944   * @throws IOException
945   */
946  public InputStream loadByCanonicalVersion(String canonical, String version) throws IOException {
947    return loadByCanonicalVersion("package", canonical, version);
948  }
949  
950  /**
951   * get a stream that contains the contents of a resource in the nominated folder, by it's canonical URL
952   * 
953   * @param folder - one of the folders in the package (main folder is "package")
954   * @param canonical - the canonical URL of the resource (exact match only)
955   * @param version - the specified version (or null if the most recent)
956   * 
957   * @return null if it is not found
958   * @throws IOException
959   */
960  public InputStream loadByCanonicalVersion(String folder, String canonical, String version) throws IOException {
961    NpmPackageFolder f = folders.get(folder);
962    List<JsonObject> matches = new ArrayList<>();
963    for (JsonObject file : f.index().getJsonObjects("files")) {
964      if (canonical.equals(file.asString("url"))) {
965        if (version != null && version.equals(file.asString("version"))) {
966          return load("package", file.asString("filename"));
967        } else if (version == null) {
968          matches.add(file);
969        }
970      }
971      if (matches.size() > 0) {
972        if (matches.size() == 1) {
973          return load("package", matches.get(0).asString("filename"));          
974        } else {
975          Collections.sort(matches, new IndexVersionSorter());
976          return load("package", matches.get(matches.size()-1).asString("filename"));          
977        }
978      }
979    }
980    return null;        
981  }
982    
983  /**
984   * get a stream that contains the contents of one of the files in the base package
985   * 
986   * @param file
987   * @return
988   * @throws IOException
989   */
990  public InputStream load(String file) throws IOException {
991    return load("package", file);
992  }
993  
994  /**
995   * get a stream that contains the contents of one of the files in a folder
996   * 
997   * @param folder
998   * @param file
999   * @return
1000   * @throws IOException
1001   */
1002  public InputStream load(String folder, String file) throws IOException {
1003    NpmPackageFolder f = folders.get(folder);
1004    if (f == null) {
1005      f = folders.get(Utilities.path("package", folder));
1006    }
1007    if (f != null && f.hasFile(file)) {
1008      return new ByteArrayInputStream(f.fetchFile(file));
1009    } else {
1010      throw new IOException("Unable to find the file "+folder+"/"+file+" in the package "+name());
1011    }
1012  }
1013
1014  /**
1015   * get a stream that contains the contents of one of the files in a folder
1016   * 
1017   * @param folder
1018   * @param file
1019   * @return
1020   * @throws IOException
1021   */
1022  public ByteProvider getProvider(String folder, String file) throws IOException {
1023    NpmPackageFolder f = folders.get(folder);
1024    if (f == null) {
1025      f = folders.get(Utilities.path("package", folder));
1026    }
1027    if (f != null && f.hasFile(file)) {
1028      return f.getProvider(file);
1029    } else {
1030      throw new IOException("Unable to find the file "+folder+"/"+file+" in the package "+name());
1031    }
1032  }
1033
1034  
1035  public boolean hasFile(String folder, String file) throws IOException {
1036    NpmPackageFolder f = folders.get(folder);
1037    if (f == null) {
1038      f = folders.get(Utilities.path("package", folder));
1039    }
1040    return f != null && f.hasFile(file);
1041  }
1042
1043
1044  /**
1045   * Handle to the package json file
1046   * 
1047   * @return
1048   */
1049  public JsonObject getNpm() {
1050    return npm;
1051  }
1052
1053  /**
1054   *  should only use this when npm == null
1055   * 
1056   * @param npm
1057   */
1058  public void setNpm(JsonObject npm) {
1059    this.npm = PackageHacker.fixPackageOnLoad(npm);
1060  }
1061
1062  /**
1063   * convenience method for getting the package name
1064   * @return
1065   */
1066  public String name() {
1067    return npm.asString("name");
1068  }
1069
1070  /**
1071   * convenience method for getting the package id (which in NPM language is the same as the name)
1072   * @return
1073   */
1074  public String id() {
1075    return npm.asString("name");
1076  }
1077
1078  public String date() {
1079    return npm.asString("date");
1080  }
1081
1082  public String canonical() {
1083    return npm.asString("canonical");
1084  }
1085
1086  /**
1087   * convenience method for getting the package version
1088   * @return
1089   */
1090  public String version() {
1091    return npm.asString("version");
1092  }
1093
1094  /**
1095   * convenience method for getting the package fhir version
1096   * @return
1097   */
1098  public String fhirVersion() {
1099    if ("hl7.fhir.core".equals(npm.asString("name")))
1100      return npm.asString("version");
1101    else if (
1102        Utilities.existsInList(npm.asString("type"), "fhir.core", "fhir.examples") &&
1103        Utilities.startsWithInList( npm.asString("name"), "hl7.fhir.r2.", "hl7.fhir.r2b.", "hl7.fhir.r3.", 
1104             "hl7.fhir.r4.", "hl7.fhir.r4b.", "hl7.fhir.r5.")) {
1105      return npm.asString("version");
1106    } else {
1107      JsonObject dep = null;
1108      if (npm.hasObject("dependencies")) {
1109        dep = npm.getJsonObject("dependencies");
1110        if (dep != null) {
1111          for (JsonProperty e : dep.getProperties()) {
1112            if (Utilities.existsInList(e.getName(), "hl7.fhir.r2.core", "hl7.fhir.r2b.core", "hl7.fhir.r3.core", "hl7.fhir.r4.core"))
1113              return e.getValue().asString();
1114            if (Utilities.existsInList(e.getName(), "hl7.fhir.core")) // while all packages are updated
1115              return e.getValue().asString();
1116          }
1117        }
1118      }
1119      if (npm.hasArray("fhirVersions")) {
1120        JsonArray e = npm.getJsonArray("fhirVersions");
1121        if (e.size() > 0) {
1122          return e.getItems().get(0).asString();
1123        }
1124      }
1125      if (dep != null) {
1126        // legacy simplifier support:
1127        if (dep.has("simplifier.core.r4"))
1128          return "4.0";
1129        if (dep.has("simplifier.core.r3"))
1130          return "3.0";
1131        if (dep.has("simplifier.core.r2"))
1132          return "2.0";
1133      }
1134      throw new FHIRException("no core dependency or FHIR Version found in the Package definition");
1135    }
1136  }
1137
1138  public String summary() {
1139    if (path != null)
1140      return path;
1141    else
1142      return "memory";
1143  }
1144
1145  public boolean isType(PackageType template) {
1146    return template.getCode().equals(type()) || template.getOldCode().equals(type()) ;
1147  }
1148
1149  public String type() {
1150    return npm.asString("type");
1151  }
1152
1153  public String description() {
1154    return npm.asString("description");
1155  }
1156
1157  public String getPath() {
1158    return path;
1159  }
1160
1161  public List<String> dependencies() {
1162    List<String> res = new ArrayList<>();
1163    if (npm.has("dependencies")) {
1164      for (JsonProperty e : npm.getJsonObject("dependencies").getProperties()) {
1165        res.add(e.getName()+"#"+e.getValue().asString());
1166      }
1167    }
1168    return res;
1169  }
1170
1171  public String homepage() {
1172    return npm.asString("homepage");
1173  }
1174
1175  public String url() {
1176    return npm.asString("url");
1177  }
1178
1179
1180  public String title() {
1181    return npm.asString("title");
1182  }
1183
1184  public String toolsVersion() {
1185    return npm.asString("tools-version");
1186  }
1187
1188  public String license() {
1189    return npm.asString("license");
1190  }
1191
1192  //  /**
1193  //   * only for use by the package manager itself
1194  //   * 
1195  //   * @param path
1196  //   */
1197  //  public void setPath(String path) {
1198  //    this.path = path;
1199  //  }
1200
1201  public String getWebLocation() {
1202    if (npm.hasPrimitive("url")) {
1203      return PackageHacker.fixPackageUrl(npm.asString("url"));
1204    } else {
1205      return npm.asString("canonical");
1206    }
1207  }
1208
1209  public InputStream loadResource(String type, String id) throws IOException {
1210    NpmPackageFolder f = folders.get("package");
1211    JsonArray files = f.index().getJsonArray("files");
1212    for (JsonElement e : files.getItems()) {
1213      JsonObject i = (JsonObject) e;
1214      if (type.equals(i.asString("resourceType")) && id.equals(i.asString("id"))) {
1215        return load("package", i.asString("filename"));
1216      }
1217    }
1218    return null;
1219  }
1220
1221  public InputStream loadExampleResource(String type, String id) throws IOException {
1222    NpmPackageFolder f = folders.get("example");
1223    if (f == null) {
1224      f = folders.get("package/example");      
1225    }
1226    if (f == null) {
1227      f = folders.get("package\\example");      
1228    }
1229    if (f != null) {
1230      JsonArray files = f.index().getJsonArray("files");
1231      for (JsonElement e : files.getItems()) {
1232        JsonObject i = (JsonObject) e;
1233        if (type.equals(i.asString("resourceType")) && id.equals(i.asString("id"))) {
1234          return load("example", i.asString("filename"));
1235        }
1236      }
1237    }
1238    return null;
1239  }
1240
1241  /** special case when playing around inside the package **/
1242  public Map<String, NpmPackageFolder> getFolders() {
1243    return folders;
1244  }
1245
1246  public void save(File directory) throws IOException {
1247    assert !minimalMemory;
1248    File dir = ManagedFileAccess.file(Utilities.path(directory.getAbsolutePath(), name()));
1249    if (!dir.exists()) {
1250      FileUtilities.createDirectory(dir.getAbsolutePath());
1251    } else {
1252      FileUtilities.clearDirectory(dir.getAbsolutePath());
1253    }
1254    
1255    for (NpmPackageFolder folder : folders.values()) {
1256      String n = folder.folderName;
1257
1258      File pd = ManagedFileAccess.file(Utilities.path(dir.getAbsolutePath(), n));
1259      if (!pd.exists()) {
1260        FileUtilities.createDirectory(pd.getAbsolutePath());
1261      }
1262      NpmPackageIndexBuilder indexer = new NpmPackageIndexBuilder();
1263      indexer.start(Utilities.path(dir.getAbsolutePath(), n, ".index.db"));
1264      for (String s : folder.content.keySet()) {
1265        byte[] b = folder.content.get(s);
1266        indexer.seeFile(s, b);
1267        if (!s.equals(".index.json") && !s.equals("package.json")) {
1268          FileUtilities.bytesToFile(b, Utilities.path(dir.getAbsolutePath(), n, s));
1269        }
1270      }
1271      byte[] cnt = indexer.build().getBytes(StandardCharsets.UTF_8);
1272      FileUtilities.bytesToFile(cnt, Utilities.path(dir.getAbsolutePath(), n, ".index.json"));
1273    }
1274    byte[] cnt = FileUtilities.stringToBytes(JsonParser.compose(npm, true));
1275    FileUtilities.bytesToFile(cnt, Utilities.path(dir.getAbsolutePath(), "package", "package.json"));
1276  }
1277  
1278  public void save(OutputStream stream) throws IOException {
1279    assert !minimalMemory;
1280    TarArchiveOutputStream tar;
1281//    ByteArrayOutputStream OutputStream;
1282//    BufferedOutputStream bufferedOutputStream;
1283    GzipCompressorOutputStream gzipOutputStream;
1284
1285//    OutputStream = new ByteArrayOutputStream();
1286//    bufferedOutputStream = new BufferedOutputStream(OutputStream);
1287    GzipParameters gp = new GzipParameters();
1288    gp.setCompressionLevel(Deflater.BEST_COMPRESSION);
1289    gzipOutputStream = new GzipCompressorOutputStream(stream, gp);
1290    tar = new TarArchiveOutputStream(gzipOutputStream);
1291
1292
1293    for (NpmPackageFolder folder : folders.values()) {
1294      String n = folder.folderName;
1295      if (!"package".equals(n) && !(n.startsWith("package/") || n.startsWith("package\\"))) {
1296        n = "package/"+n;
1297      }
1298      NpmPackageIndexBuilder indexer = new NpmPackageIndexBuilder();
1299      String filename = Utilities.path("[tmp]", "tmp-" + UUID.randomUUID() + ".db");
1300      indexer.start(filename);
1301      for (String s : folder.content.keySet()) {
1302        byte[] b = folder.content.get(s);
1303        String name = n+"/"+s;
1304        if (b == null) {
1305          log.warn(name+" is null");
1306        } else {
1307          indexer.seeFile(s, b);
1308          if (!s.equals(".index.json") && !s.equals(".index.db") && !s.equals("package.json")) {
1309            TarArchiveEntry entry = new TarArchiveEntry(name);
1310            entry.setSize(b.length);
1311            tar.putArchiveEntry(entry);
1312            tar.write(b);
1313            tar.closeArchiveEntry();
1314          }
1315        }
1316      }
1317      byte[] cnt = indexer.build().getBytes(StandardCharsets.UTF_8);
1318      TarArchiveEntry entry = new TarArchiveEntry(n+"/.index.json");
1319      entry.setSize(cnt.length);
1320      tar.putArchiveEntry(entry);
1321      tar.write(cnt);
1322      tar.closeArchiveEntry();
1323      var file = ManagedFileAccess.file(filename);
1324      if (file.exists()) {
1325        cnt = FileUtilities.fileToBytes(file);
1326        file.delete();
1327        entry = new TarArchiveEntry(n+"/.index.db");
1328        entry.setSize(cnt.length);
1329        tar.putArchiveEntry(entry);
1330        tar.write(cnt);
1331        tar.closeArchiveEntry();
1332      }
1333    }
1334    byte[] cnt = FileUtilities.stringToBytes(JsonParser.compose(npm, true));
1335    TarArchiveEntry entry = new TarArchiveEntry("package/package.json");
1336    entry.setSize(cnt.length);
1337    tar.putArchiveEntry(entry);
1338    tar.write(cnt);
1339    tar.closeArchiveEntry();
1340
1341    tar.finish();
1342    tar.close();
1343    gzipOutputStream.close();
1344//    bufferedOutputStream.close();
1345//    OutputStream.close();
1346//    byte[] b = OutputStream.toByteArray();
1347//    stream.write(b);
1348  }
1349
1350  /**
1351   * Keys are resource type names, values are filenames
1352   */
1353  public Map<String, List<String>> getTypes() {
1354    return folders.get("package").types;
1355  }
1356
1357  public String fhirVersionList() {
1358    if (npm.has("fhirVersions")) {
1359      CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
1360      if (npm.hasArray("fhirVersions")) {
1361        for (String n : npm.getJsonArray("fhirVersions").asStrings()) {
1362          b.append(n);
1363        }
1364      }
1365      if (npm.hasPrimitive("fhirVersions")) {
1366        b.append(npm.asString("fhirVersions"));
1367      }
1368      return b.toString();
1369    } else
1370      return "";
1371  }
1372
1373  public String dependencySummary() {
1374    if (npm.has("dependencies")) {
1375      CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
1376      for (JsonProperty e : npm.getJsonObject("dependencies").getProperties()) {
1377        b.append(e.getName()+"#"+e.getValue().asString());
1378      }
1379      return b.toString();
1380    } else
1381      return "";
1382  }
1383
1384  public void unPack(String dir) throws IOException {
1385    unPack (dir, false, new ArrayList<>());
1386  }
1387
1388  public void unPackWithAppend(String dir, List<String> files) throws IOException {
1389    unPack (dir, true, files);
1390  }
1391
1392  public void unPack(String dir, boolean withAppend) throws IOException {
1393    unPack (dir, withAppend, new ArrayList<>());
1394  }
1395  
1396  public void unPack(String dir, boolean withAppend, List<String> files) throws IOException {
1397    assert !minimalMemory;
1398    
1399    for (NpmPackageFolder folder : folders.values()) {
1400      String dn = folder.getFolderName();
1401      if (!dn.equals("package") && (dn.startsWith("package/") || dn.startsWith("package\\"))) {
1402        dn = dn.substring(8);
1403      }
1404      if (dn.equals("$root")) {
1405        dn = dir;
1406      } else {
1407         dn = Utilities.path(dir, dn);
1408      }
1409      FileUtilities.createDirectory(dn);
1410      for (String s : folder.listFiles()) {
1411        String fn = Utilities.path(dn, s);
1412        File f = ManagedFileAccess.file(fn);
1413        if (withAppend && f.getName().startsWith("_append.")) {
1414          String appendFn = Utilities.path(dn, s.substring(8));
1415          f = ManagedFileAccess.file(appendFn);
1416          files.add(f.getAbsolutePath());
1417          if (f.exists())
1418            FileUtilities.appendBytesToFile(folder.fetchFile(s), appendFn);        
1419          else
1420            FileUtilities.bytesToFile(folder.fetchFile(s), appendFn);        
1421        } else
1422          files.add(f.getAbsolutePath());
1423          FileUtilities.bytesToFile(folder.fetchFile(s), fn);
1424      }      
1425    }
1426  }
1427
1428  private List<String> sorted(Set<String> keys) {
1429    List<String> res = new ArrayList<String>();
1430    res.addAll(keys);
1431    Collections.sort(res);
1432    return res ;
1433  }
1434
1435  public void clearFolder(String folderName) {
1436    NpmPackageFolder folder = folders.get(folderName);
1437    folder.content.clear();
1438    folder.types.clear();    
1439  }
1440
1441  public void deleteFolder(String folderName) {
1442    folders.remove(folderName);
1443  }
1444
1445  public void addFile(String folderName, String name, byte[] cnt, String type) {
1446    assert !minimalMemory;
1447    
1448    if (!folders.containsKey(folderName)) {
1449      folders.put(folderName, new NpmPackageFolder(folderName));
1450    }
1451    NpmPackageFolder folder = folders.get(folderName);
1452    folder.content.put(name, cnt);
1453    if (!folder.types.containsKey(type))
1454      folder.types.put(type, new ArrayList<>());
1455    folder.types.get(type).add(name);
1456    if ("package".equals(folderName) && "package.json".equals(name)) {
1457      try {
1458        npm = PackageHacker.fixPackageOnLoad(JsonParser.parseObject(cnt));
1459      } catch (IOException e) {
1460      }
1461    }
1462  }
1463
1464  public void loadAllFiles() throws IOException {
1465    for (String folder : folders.keySet()) {
1466      NpmPackageFolder pf = folders.get(folder);
1467      String p = folder.contains("$") ? path : Utilities.path(path, folder);
1468      File file = ManagedFileAccess.file(p);
1469      if (file.exists()) {
1470        for (File f : file.listFiles()) {
1471          if (!f.isDirectory() && !isInternalExemptFile(f)) {
1472            pf.getContent().put(f.getName(), FileUtilities.fileToBytes(f));
1473          }
1474        }
1475      }
1476    }
1477  }
1478
1479  public void loadAllFiles(ITransformingLoader loader) throws IOException {
1480    for (String folder : folders.keySet()) {
1481      NpmPackageFolder pf = folders.get(folder);
1482      String p = folder.contains("$") ? path : Utilities.path(path, folder);
1483      for (File f : ManagedFileAccess.file(p).listFiles()) {
1484        if (!f.isDirectory() && !isInternalExemptFile(f)) {
1485          pf.getContent().put(f.getName(), loader.load(f));
1486        }
1487      }
1488    }
1489  }
1490
1491  public boolean isChangedByLoader() {
1492    return changedByLoader;
1493  }
1494
1495  public boolean isCore() {
1496    return Utilities.existsInList(npm.asString("type"), "fhir.core", "Core");
1497  }
1498
1499  public boolean isCoreExamples() {
1500    return name().startsWith("hl7.fhir.r") && name().endsWith(".examples");
1501  }
1502  
1503  
1504  public boolean isTx() {
1505    return npm.asString("name").startsWith("hl7.terminology");
1506  }
1507
1508  public boolean hasCanonical(String url) throws IOException {
1509    if (url == null) {
1510      return false;
1511    }
1512    String u = url.contains("|") ?  url.substring(0, url.indexOf("|")) : url;
1513    String v = url.contains("|") ?  url.substring(url.indexOf("|")+1) : null;
1514    NpmPackageFolder folder = folders.get("package");
1515    if (folder != null) {
1516      for (JsonObject o : folder.index().getJsonObjects("files")) {
1517        if (u.equals(o.asString("url"))) {
1518          if (v == null || v.equals(o.asString("version"))) {
1519            return true;
1520          }
1521        }
1522      }
1523    }
1524    return false;
1525  }
1526
1527  public boolean canLazyLoad() throws IOException {
1528    for (NpmPackageFolder folder : folders.values()) {
1529      if (folder.folder == null) {        
1530        return false;
1531      }
1532    }
1533    if (Utilities.existsInList(name(), "fhir.test.data.r2", "fhir.test.data.r3", "fhir.test.data.r4", "fhir.tx.support.r2", "fhir.tx.support.r3", "fhir.tx.support.r4", "us.nlm.vsac")) {
1534      return true;
1535    }
1536    if (npm.asBoolean("lazy-load")) {
1537      return true;
1538    }
1539    return hasFile("other", "spec.internals") || folders.get("package").cachedIndex != null;
1540  }
1541
1542  public boolean isNotForPublication() {
1543    return npm.asBoolean("notForPublication");
1544 }
1545
1546  public InputStream load(PackageResourceInformation p) throws IOException {
1547    if (p.filename.startsWith("@")) {
1548      String[] pl = p.filename.substring(1).split("\\/");
1549      return new ByteArrayInputStream(folders.get(pl[0]).content.get(pl[1]));
1550    } else {
1551      return ManagedFileAccess.inStream(p.filename);
1552    }
1553  }
1554
1555  public Date dateAsDate() {
1556    try {
1557      String d = date();
1558      if (d == null) {
1559        switch (name()) {
1560        case "hl7.fhir.r2.core":  d = "20151024000000"; break;
1561        case "hl7.fhir.r2b.core": d = "20160330000000"; break;
1562        case "hl7.fhir.r3.core":  d = "20191024000000"; break;
1563        case "hl7.fhir.r4.core":  d = "20191030000000"; break;
1564        case "hl7.fhir.r4b.core": d = "202112200000000"; break;
1565        case "hl7.fhir.r5.core":  d = "20211219000000"; break;
1566        default:
1567          return new Date();
1568        }
1569      }
1570      return new SimpleDateFormat("yyyyMMddHHmmss").parse(d);
1571    } catch (ParseException e) {
1572      // this really really shouldn't happen
1573      return new Date();
1574    }
1575  }
1576
1577  public static NpmPackage fromUrl(String source) throws IOException {
1578    HTTPResult res = ManagedWebAccess.get(Arrays.asList("npm-package", "fhir-package"), source+"?nocache=" + System.currentTimeMillis());
1579    res.checkThrowException();
1580    return fromPackage(new ByteArrayInputStream(res.getContent()));
1581  }
1582
1583  @Override
1584  public String toString() {
1585    return "NpmPackage "+name()+"#"+version()+" [path=" + path + "]";
1586  }
1587
1588  public String getFilePath(String d) throws IOException {
1589    return Utilities.path(path, "package", d);
1590  }
1591
1592  public boolean isMinimalMemory() {
1593    return minimalMemory;
1594  }
1595
1596  public int getSize() {
1597    return size;
1598  }
1599
1600  public void setSize(int size) {
1601    this.size = size;
1602  }
1603
1604  public boolean isWarned() {
1605    return warned;
1606  }
1607
1608  public void setWarned(boolean warned) {
1609    this.warned = warned;
1610  }
1611
1612  public String vid() {
1613    return id()+"#"+version();
1614  }
1615
1616  public static boolean isLoadCustomResources() {
1617    return loadCustomResources;
1618  }
1619
1620  public static void setLoadCustomResources(boolean loadCustomResources) {
1621    NpmPackage.loadCustomResources = loadCustomResources;
1622  }
1623
1624  public LocalDate dateAsLocalDate() {
1625    String date = date();
1626    String d = date.substring(0, 4)+"-"+date.substring(4, 6)+"-"+date.substring(6, 8);
1627    return LocalDate.parse(d);
1628  }
1629
1630}