001package org.hl7.fhir.utilities;
002
003import java.io.File;
004import java.io.FileInputStream;
005import java.io.FileNotFoundException;
006import java.io.FileOutputStream;
007import java.io.FilenameFilter;
008import java.io.IOException;
009import java.io.InputStream;
010import java.io.OutputStream;
011import java.nio.charset.StandardCharsets;
012import java.nio.file.Files;
013import java.nio.file.Path;
014import java.nio.file.Paths;
015import java.nio.file.StandardCopyOption;
016import java.nio.file.StandardOpenOption;
017import java.util.ArrayList;
018import java.util.Arrays;
019import java.util.List;
020import java.util.regex.Pattern;
021
022import org.apache.commons.io.FileUtils;
023import org.hl7.fhir.exceptions.FHIRException;
024import org.hl7.fhir.utilities.FileNotifier.FileNotifier2;
025import org.hl7.fhir.utilities.filesystem.CSFile;
026import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
027
028
029public class FileUtilities {
030
031  public static String bytesToString(final byte[] bs) throws IOException {
032    return new String(bs, StandardCharsets.UTF_8);
033  }
034
035  public static String bytesToString(final byte[] bs, final boolean removeBOM) throws IOException {
036    final String read = new String(bs, StandardCharsets.UTF_8);
037    if (removeBOM)
038      return read.replace("\uFEFF", "");
039    else
040      return read;
041  }
042
043  public static void bytesToFile(final byte[] bytes, final String path) throws IOException {
044    try (final OutputStream sw = ManagedFileAccess.outStream(ManagedFileAccess.csfile(path))) {
045      if (bytes != null) {
046        sw.write(bytes);
047      }
048    }
049  }
050  
051  public static void bytesToFile(final byte[] bytes, final File f) throws IOException {
052    try (final OutputStream sw = ManagedFileAccess.outStream(f)) {
053      sw.write(bytes);
054    }
055  }
056  
057  public static void appendBytesToFile(final byte[] bytes, final String path) throws IOException {
058    byte[] linebreak = new byte[] {13, 10};
059    Files.write(Paths.get(path), linebreak, StandardOpenOption.APPEND);
060    Files.write(Paths.get(path), bytes, StandardOpenOption.APPEND);
061  }
062
063  public static void stringToStream(final String content, final OutputStream stream) throws IOException {
064    stream.write(content.getBytes(StandardCharsets.UTF_8));
065  }
066  
067  public static byte[] stringToBytes(final String content) throws IOException {
068    return content.getBytes(StandardCharsets.UTF_8);
069  }
070
071  public static void stringToFile(final String content, final String path) throws IOException  {
072    final File file = ManagedFileAccess.csfile(path);
073    stringToFile(content, file);
074  }
075
076  public static void stringToFileIfDifferent(final String content, final String path) throws IOException  {
077    final File file = ManagedFileAccess.csfile(path);
078    String current = file.exists() ? fileToString(file) : null;
079    if (current == null || !current.equals(content)) {
080      stringToFile(content, file);
081    }
082  }
083
084  public static void stringToFile(final String content, final File file) throws IOException {
085    try (final OutputStream output = Files.newOutputStream(file.toPath())) {
086      output.write(content.getBytes(StandardCharsets.UTF_8));
087    }
088  }  
089  
090  public static void stringToFileWithBOM(final String content, final File file) throws IOException  {
091    try (final OutputStream output = Files.newOutputStream(file.toPath())) {
092      output.write(new byte[]{(byte)239, (byte)187, (byte)191});
093      output.write(content.getBytes(StandardCharsets.UTF_8));
094    }
095  }
096  
097  public static void stringToFileWithBOM(final String content, final String path) throws IOException  {
098    final File file = ManagedFileAccess.csfile(path);
099    stringToFileWithBOM(content, file);
100  }
101  
102  public static String fileToString(final File f) throws FileNotFoundException, IOException {
103    // Files.readString(Path) will fail on invalid UTF-8 byte sequences, so we use Files.readAllBytes() instead.
104    // This would happen when reading an XSLX file, for example.
105    return new String(Files.readAllBytes(f.toPath()), StandardCharsets.UTF_8);
106  }
107  
108  public static String fileToString(final String src) throws FileNotFoundException, IOException  {
109    final CSFile f = ManagedFileAccess.csfile(src);
110    if (!f.exists()) {
111      throw new IOException("File "+src+" not found");
112    }
113    return fileToString(f);
114  }
115
116  public static byte[] fileToBytes(final String srcFile) throws FileNotFoundException, IOException {
117    final File f = ManagedFileAccess.csfile(srcFile);
118    return Files.readAllBytes(f.toPath());
119  }
120
121  public static byte[] fileToBytes(final File file) throws FileNotFoundException, IOException {
122    return Files.readAllBytes(file.toPath());
123  }
124
125  /**
126   * 
127   * fileToBytes insists in case correctness to ensure that stuff works across linux and windows, but it's not always appropriate to check case (e.g. validator parameters)
128   * 
129   * @param srcFile
130   * @return
131   * @throws FileNotFoundException
132   * @throws IOException
133   */
134  public static byte[] fileToBytesNCS(final String srcFile) throws FileNotFoundException, IOException {
135    return Files.readAllBytes(Path.of(srcFile));
136  }
137
138  public static List<String> fileToLines(String file) throws FileNotFoundException, IOException {
139    Pattern LINE_SEP_PATTERN = Pattern.compile("\\R");
140    List<String> res = new ArrayList<String>();
141    for (String s : LINE_SEP_PATTERN.split(fileToString(file))) {
142      res.add(s);
143    }
144    return res;
145  }
146
147  public static String[] fileToLines(File file) throws FileNotFoundException, IOException {
148    Pattern LINE_SEP_PATTERN = Pattern.compile("\\R");
149    return LINE_SEP_PATTERN.split(fileToString(file));
150  }
151
152  public static String[] streamToLines(InputStream stream) throws FileNotFoundException, IOException {
153    Pattern LINE_SEP_PATTERN = Pattern.compile("\\R");
154    return LINE_SEP_PATTERN.split(streamToString(stream));
155  }
156
157  public static String streamToString(final InputStream input) throws IOException  {
158    return new String(input.readAllBytes(), StandardCharsets.UTF_8).replace("\uFEFF", "");
159  }
160
161  public static byte[] streamToBytes(final InputStream input) throws IOException  {
162    if (input == null) {
163      return null;
164    }
165    final byte[] read = input.readAllBytes();
166    input.close();
167    return read;
168  }
169
170  public static byte[] streamToBytesNoClose(final InputStream input) throws IOException  {
171    if (input == null) {
172      return null;
173    }
174    return input.readAllBytes();
175  }
176
177  public static void streamToFile(final InputStream stream, final String filename) throws IOException {
178    Files.copy(stream, Path.of(filename), StandardCopyOption.REPLACE_EXISTING);
179    stream.close();
180  }
181
182  public static void streamToFileNoClose(final InputStream stream, final String filename) throws IOException {
183    Files.copy(stream, Path.of(filename), StandardCopyOption.REPLACE_EXISTING);
184  }
185        
186  public static void linesToFile(File f, String[] lines) throws IOException {
187    Files.write(f.toPath(), List.of(lines), StandardCharsets.UTF_8);
188  }
189  
190  public static void linesToFile(String path, String[] lines) throws IOException  {
191    final File file = ManagedFileAccess.csfile(path);
192    Files.write(file.toPath(), List.of(lines), StandardCharsets.UTF_8);
193  }
194  
195  public static void linesToFile(String path, List<String> lines) throws IOException  {
196    final File file = ManagedFileAccess.csfile(path);
197    Files.write(file.toPath(), lines, StandardCharsets.UTF_8);
198  }
199  
200  public static String fileTitle(String file) throws IOException {
201    if (file == null)
202      return null;
203    String s = ManagedFileAccess.file(file).getName();
204    return s.indexOf(".") == -1 ? s : s.substring(0, s.indexOf("."));
205  }
206
207  public static String getDirectoryForFile(String filepath) throws IOException {
208    File f = ManagedFileAccess.file(filepath);
209    return f.getParent();
210  }
211
212  public static String getDirectoryForFile(File file) throws IOException {
213    return file.getParent();
214  }
215
216  public static String getRelativePath(String root, String path) {
217    String res = path.substring(root.length());
218    if (res.startsWith(File.separator)) {
219      res = res.substring(1);
220    }
221    return res;
222  }
223
224  public static File createDirectory(String path) throws IOException {
225    ManagedFileAccess.csfile(path).mkdirs();
226    return ManagedFileAccess.file(path);
227  }
228
229  public static File createDirectoryNC(String path) throws IOException {
230    ManagedFileAccess.file(path).mkdirs();
231    return ManagedFileAccess.file(path);
232  }
233
234  public static void copyFile(File sourceFile, File destFile) throws IOException {
235    if (!destFile.exists()) {
236      if (!ManagedFileAccess.csfile(destFile.getParent()).exists()) {
237        createDirectory(destFile.getParent());
238      }
239      destFile.createNewFile();
240    } else if (!destFile.getCanonicalFile().getName().equals(destFile.getName())) {
241      // case mismatch
242      destFile.delete();
243      destFile.createNewFile();
244    }
245  
246    FileInputStream source = null;
247    FileOutputStream destination = null;
248  
249    try {
250      source = ManagedFileAccess.inStream(sourceFile);
251      destination = ManagedFileAccess.outStream(destFile);
252      destination.getChannel().transferFrom(source.getChannel(), 0, source.getChannel().size());
253    } finally {
254      if (source != null) {
255        source.close();
256      }
257      if (destination != null) {
258        destination.close();
259      }
260    }
261  }
262
263  public static void copyFile(String source, String dest) throws IOException {
264    copyFile(ManagedFileAccess.file(source), ManagedFileAccess.file(dest));
265  }
266
267  public static void copyDirectory(String sourceFolder, String destFolder, FileNotifier notifier) throws IOException, FHIRException {
268    CSFile src = ManagedFileAccess.csfile(sourceFolder);
269    if (!src.exists())
270      throw new FHIRException("Folder " + sourceFolder + " not found");
271    File dst = ManagedFileAccess.file(destFolder);
272    if(!dst.getCanonicalFile().getName().equals(dst.getName())) {
273      File tmp = ManagedFileAccess.file(destFolder+System.currentTimeMillis());
274      if (!dst.renameTo(tmp)) {
275        throw new IOException("fixing case from "+dst.getCanonicalFile().getName()+" to "+tmp.getName()+" failed");
276      }
277      if (!tmp.renameTo(dst)) {
278        throw new IOException("fixing case from "+tmp.getCanonicalFile().getName()+" to "+dst.getName()+" failed");
279      }
280    } else if (!dst.exists()) {    
281      createDirectory(destFolder);
282    }
283  
284    String[] files = src.list();
285    for (String f : files) {
286      if (ManagedFileAccess.csfile(sourceFolder + File.separator + f).isDirectory()) {
287        if (!f.startsWith(".")) { // ignore .git files...
288          copyDirectory(sourceFolder + File.separator + f, destFolder + File.separator + f, notifier);
289        }
290      } else {
291        if (notifier != null)
292          notifier.copyFile(sourceFolder + File.separator + f, destFolder + File.separator + f);
293        FileUtilities.copyFile(ManagedFileAccess.csfile(sourceFolder + File.separator + f), new /*CS*/File(destFolder + File.separator + f)); // case doesn't have to match on the target
294      }
295    }
296  }
297
298  public static void copyDirectory2(String sourceFolder, String destFolder, FileNotifier2 notifier) throws IOException, FHIRException {
299    CSFile src = ManagedFileAccess.csfile(sourceFolder);
300    if (!src.exists())
301      throw new FHIRException("Folder " + sourceFolder + " not found");
302    createDirectory(destFolder);
303  
304    String[] files = src.list();
305    for (String f : files) {
306      if (ManagedFileAccess.csfile(sourceFolder + File.separator + f).isDirectory()) {
307        if (!f.startsWith(".")) { // ignore .git files...
308          boolean doCopy = notifier != null ? notifier.copyFolder(sourceFolder + File.separator + f, destFolder + File.separator + f) : true;
309          if (doCopy) {
310            copyDirectory2(sourceFolder + File.separator + f, destFolder + File.separator + f, notifier);
311          }
312        }
313      } else {
314        boolean doCopy = notifier != null ? notifier.copyFile(sourceFolder + File.separator + f, destFolder + File.separator + f) : true;
315        if (doCopy) {
316          FileUtilities.copyFile(ManagedFileAccess.csfile(sourceFolder + File.separator + f), ManagedFileAccess.csfile(destFolder + File.separator + f));
317        }
318      }
319    }
320  }
321
322  public static void copyFileToDirectory(File source, File destDir) throws IOException {
323    FileUtilities.copyFile(source, ManagedFileAccess.file(Utilities.path(destDir.getAbsolutePath(), source.getName())));
324  }
325
326  public static void renameDirectory(String source, String dest) throws FHIRException, IOException {
327    File src = ManagedFileAccess.file(source);
328    File dst = ManagedFileAccess.file(dest);
329    if (!src.renameTo(dst)) {
330      int i = 0;
331      do {
332        try {
333          Thread.sleep(20);
334        } catch (Exception e) {
335          // nothing
336        }
337        System.gc();
338        i++;
339      } while (!src.renameTo(dst) && i < 10);
340      if (src.exists()) {
341        copyDirectory(source, dest, null);
342        try {
343          src.delete(); 
344        } catch (Exception e) {
345          // nothing
346        }
347      }
348    }
349  
350  }
351
352  public static List<String> listAllFiles(String path, List<String> ignoreList) throws IOException {
353    List<String> res = new ArrayList<>();
354    scanForFiles(res, path, ManagedFileAccess.file(path), ignoreList);
355    return res;
356  }
357  
358  private static void scanForFiles(List<String> res, String root, File dir, List<String> ignoreList) {
359    for (File f : dir.listFiles()) {
360      if (ignoreList == null || !ignoreList.contains(f.getAbsolutePath())) {
361        if (f.isDirectory()) {
362          scanForFiles(res, root, f, ignoreList);
363        } else if (!f.getName().equals(".DS_Store")) {
364          res.add(getRelativePath(root, f.getAbsolutePath()));
365        }
366      }
367    }
368  }
369
370  /**
371   * Delete a directory atomically by first renaming it to a temp directory in
372   * its parent, and then deleting its contents.
373   *
374   * @param path The directory to delete.
375   */
376  public static void atomicDeleteDirectory(String path) throws IOException {
377
378      File directory = ManagedFileAccess.file(path);
379
380      String tempDirectoryPath = Utilities.generateUniqueRandomUUIDPath(directory.getParent());
381      File tempDirectory = ManagedFileAccess.file(tempDirectoryPath);
382     if (!directory.renameTo(tempDirectory)) {
383       throw new IOException("Unable to rename directory " + path + " to " + tempDirectory +" for atomic delete");
384     }
385     clearDirectory(tempDirectory.getAbsolutePath());
386     if (!tempDirectory.delete()) {
387       throw new IOException("Unable to delete temp directory " + tempDirectory + " when atomically deleting " + path);
388     }
389  }
390
391  public interface FileVisitor {
392    void visitFile(File file) throws FileNotFoundException, IOException;
393  }
394
395  public static void visitFiles(String folder, String extension, FileVisitor visitor) throws FileNotFoundException, IOException {
396    visitFiles(ManagedFileAccess.file(folder), extension, visitor);
397  }
398
399  public static void visitFiles(File folder, String extension, FileVisitor visitor) throws FileNotFoundException, IOException {
400    for (File file : folder.listFiles()) {
401      if (file.isDirectory())
402        visitFiles(file, extension, visitor);
403      else if (extension == null || file.getName().endsWith(extension))
404        visitor.visitFile(file);
405    }
406  }
407
408  public static int countFilesInDirectory(String dirName) throws IOException {
409    File dir = ManagedFileAccess.file(dirName);
410    if (dir.exists() == false) {
411      return 0;
412    }
413    int i = 0;
414    for (File f : dir.listFiles())
415      if (!f.isDirectory())
416        i++;
417    return i;
418  }
419
420  public static boolean checkFileExists(String purpose, String dir, String file, List<String> errors)
421      throws IOException {
422    if (!ManagedFileAccess.csfile(dir + file).exists()) {
423      if (errors != null)
424        errors.add("Unable to find " + purpose + " file " + file + " in " + dir);
425      return false;
426    } else {
427      return true;
428    }
429  }
430
431  public static boolean checkFolderExists(String dir, List<String> errors)
432      throws IOException {
433    if (!ManagedFileAccess.csfile(dir).exists()) {
434      errors.add("Unable to find directory " + dir);
435      return false;
436    } else {
437      return true;
438    }
439  }
440
441  public static File createTempFile(String prefix, String suffix) throws IOException {
442    // this allows use to eaily identify all our dtemp files and delete them, since delete on Exit doesn't really work.
443    File file = File.createTempFile("ohfu-" + prefix, suffix);
444    file.deleteOnExit();
445    return file;
446  }
447
448  public static void deleteTempFiles() throws IOException {
449    File file = createTempFile("test", "test");
450    String folder = getDirectoryForFile(file.getAbsolutePath());
451    String[] list = ManagedFileAccess.file(folder).list(new FilenameFilter() {
452      public boolean accept(File dir, String name) {
453        return name.startsWith("ohfu-");
454      }
455    });
456    if (list != null) {
457      for (String n : list) {
458        ManagedFileAccess.file(Utilities.path(folder, n)).delete();
459      }
460    }
461  }
462
463  public static void clearDirectory(String folder, String... exemptions) throws IOException {
464    File dir = ManagedFileAccess.file(folder);
465    if (dir.exists()) {
466      if (exemptions.length == 0)
467        FileUtils.cleanDirectory(dir);
468      else {
469        String[] files = ManagedFileAccess.csfile(folder).list();
470        if (files != null) {
471          for (String f : files) {
472            if (!Utilities.existsInList(f, exemptions)) {
473              File fh = ManagedFileAccess.csfile(folder + File.separatorChar + f);
474              if (fh.isDirectory())
475                clearDirectory(fh.getAbsolutePath());
476              fh.delete();
477            }
478          }
479        }
480      }
481    }
482  }
483  
484  public static void deleteAllFiles(String folder, String type) throws IOException {
485    File src = ManagedFileAccess.file(folder);
486    String[] files = src.list();
487    for (String f : files) {
488      if (ManagedFileAccess.file(folder + File.separator + f).isDirectory()) {
489        deleteAllFiles(folder + File.separator + f, type);
490      } else if (f.endsWith(type)) {
491        ManagedFileAccess.file(folder + File.separator + f).delete();
492      }
493    }
494  
495  }
496
497  public static String changeFileExt(String name, String ext) {
498    if (name.lastIndexOf('.') > -1)
499      return name.substring(0, name.lastIndexOf('.')) + ext;
500    else
501      return name + ext;
502  }
503
504  public static boolean isIgnorableFile(File file) {
505    return Utilities.existsInList(file.getName(), ".DS_Store");
506  }
507
508  public static void deleteEmptyFolders(File df) {
509    for (File f : df.listFiles()) {
510      if (f.isDirectory()) {
511        deleteEmptyFolders(f);
512      }
513    }
514    boolean empty = true;
515    for (File f : df.listFiles()) {
516      empty = false;
517      break;
518    }
519    if (empty) {
520      df.delete();
521    }
522  }
523
524  public static String cleanFileName(String badFileName) {
525    StringBuilder cleanName = new StringBuilder();
526    int len = badFileName.codePointCount(0, badFileName.length());
527    for (int i=0; i<len; i++) {
528      int c = badFileName.codePointAt(i);
529      if (Arrays.binarySearch(Utilities.illegalChars, c) < 0) {
530        cleanName.appendCodePoint(c);
531      }
532    }
533    return cleanName.toString();
534  }
535
536  public static void copyFiles(String source, String dest, String... extensions) throws IOException {
537    for (File f : new File(source).listFiles()) {
538      boolean copy = false;
539      for (String e : extensions) {
540        if (f.getName().endsWith(e)) {
541          copy = true;
542        }
543      }
544      if (copy) {
545        FileUtilities.copyFile(f.getAbsolutePath(), Utilities.path(dest, f.getName()));
546      }
547    }
548    
549  }
550
551  public static boolean isEmptyDirectory(File dst) {
552    File[] files = dst.listFiles();
553    return files == null || files.length == 0;
554  }
555
556}