001package org.hl7.fhir.utilities; 002 003import java.io.File; 004import java.io.FileInputStream; 005import java.io.FileNotFoundException; 006import java.io.FileOutputStream; 007import java.io.FilenameFilter; 008import java.io.IOException; 009import java.io.InputStream; 010import java.io.OutputStream; 011import java.nio.charset.StandardCharsets; 012import java.nio.file.Files; 013import java.nio.file.Path; 014import java.nio.file.Paths; 015import java.nio.file.StandardCopyOption; 016import java.nio.file.StandardOpenOption; 017import java.util.ArrayList; 018import java.util.Arrays; 019import java.util.List; 020import java.util.regex.Pattern; 021 022import org.apache.commons.io.FileUtils; 023import org.hl7.fhir.exceptions.FHIRException; 024import org.hl7.fhir.utilities.FileNotifier.FileNotifier2; 025import org.hl7.fhir.utilities.filesystem.CSFile; 026import org.hl7.fhir.utilities.filesystem.ManagedFileAccess; 027 028 029public class FileUtilities { 030 031 public static String bytesToString(final byte[] bs) throws IOException { 032 return new String(bs, StandardCharsets.UTF_8); 033 } 034 035 public static String bytesToString(final byte[] bs, final boolean removeBOM) throws IOException { 036 final String read = new String(bs, StandardCharsets.UTF_8); 037 if (removeBOM) 038 return read.replace("\uFEFF", ""); 039 else 040 return read; 041 } 042 043 public static void bytesToFile(final byte[] bytes, final String path) throws IOException { 044 try (final OutputStream sw = ManagedFileAccess.outStream(ManagedFileAccess.csfile(path))) { 045 if (bytes != null) { 046 sw.write(bytes); 047 } 048 } 049 } 050 051 public static void bytesToFile(final byte[] bytes, final File f) throws IOException { 052 try (final OutputStream sw = ManagedFileAccess.outStream(f)) { 053 sw.write(bytes); 054 } 055 } 056 057 public static void appendBytesToFile(final byte[] bytes, final String path) throws IOException { 058 byte[] linebreak = new byte[] {13, 10}; 059 Files.write(Paths.get(path), linebreak, StandardOpenOption.APPEND); 060 Files.write(Paths.get(path), bytes, StandardOpenOption.APPEND); 061 } 062 063 public static void stringToStream(final String content, final OutputStream stream) throws IOException { 064 stream.write(content.getBytes(StandardCharsets.UTF_8)); 065 } 066 067 public static byte[] stringToBytes(final String content) throws IOException { 068 return content.getBytes(StandardCharsets.UTF_8); 069 } 070 071 public static void stringToFile(final String content, final String path) throws IOException { 072 final File file = ManagedFileAccess.csfile(path); 073 stringToFile(content, file); 074 } 075 076 public static void stringToFileIfDifferent(final String content, final String path) throws IOException { 077 final File file = ManagedFileAccess.csfile(path); 078 String current = file.exists() ? fileToString(file) : null; 079 if (current == null || !current.equals(content)) { 080 stringToFile(content, file); 081 } 082 } 083 084 public static void stringToFile(final String content, final File file) throws IOException { 085 try (final OutputStream output = Files.newOutputStream(file.toPath())) { 086 output.write(content.getBytes(StandardCharsets.UTF_8)); 087 } 088 } 089 090 public static void stringToFileWithBOM(final String content, final File file) throws IOException { 091 try (final OutputStream output = Files.newOutputStream(file.toPath())) { 092 output.write(new byte[]{(byte)239, (byte)187, (byte)191}); 093 output.write(content.getBytes(StandardCharsets.UTF_8)); 094 } 095 } 096 097 public static void stringToFileWithBOM(final String content, final String path) throws IOException { 098 final File file = ManagedFileAccess.csfile(path); 099 stringToFileWithBOM(content, file); 100 } 101 102 public static String fileToString(final File f) throws FileNotFoundException, IOException { 103 // Files.readString(Path) will fail on invalid UTF-8 byte sequences, so we use Files.readAllBytes() instead. 104 // This would happen when reading an XSLX file, for example. 105 return new String(Files.readAllBytes(f.toPath()), StandardCharsets.UTF_8); 106 } 107 108 public static String fileToString(final String src) throws FileNotFoundException, IOException { 109 final CSFile f = ManagedFileAccess.csfile(src); 110 if (!f.exists()) { 111 throw new IOException("File "+src+" not found"); 112 } 113 return fileToString(f); 114 } 115 116 public static byte[] fileToBytes(final String srcFile) throws FileNotFoundException, IOException { 117 final File f = ManagedFileAccess.csfile(srcFile); 118 return Files.readAllBytes(f.toPath()); 119 } 120 121 public static byte[] fileToBytes(final File file) throws FileNotFoundException, IOException { 122 return Files.readAllBytes(file.toPath()); 123 } 124 125 /** 126 * 127 * fileToBytes insists in case correctness to ensure that stuff works across linux and windows, but it's not always appropriate to check case (e.g. validator parameters) 128 * 129 * @param srcFile 130 * @return 131 * @throws FileNotFoundException 132 * @throws IOException 133 */ 134 public static byte[] fileToBytesNCS(final String srcFile) throws FileNotFoundException, IOException { 135 return Files.readAllBytes(Path.of(srcFile)); 136 } 137 138 public static List<String> fileToLines(String file) throws FileNotFoundException, IOException { 139 Pattern LINE_SEP_PATTERN = Pattern.compile("\\R"); 140 List<String> res = new ArrayList<String>(); 141 for (String s : LINE_SEP_PATTERN.split(fileToString(file))) { 142 res.add(s); 143 } 144 return res; 145 } 146 147 public static String[] fileToLines(File file) throws FileNotFoundException, IOException { 148 Pattern LINE_SEP_PATTERN = Pattern.compile("\\R"); 149 return LINE_SEP_PATTERN.split(fileToString(file)); 150 } 151 152 public static String[] streamToLines(InputStream stream) throws FileNotFoundException, IOException { 153 Pattern LINE_SEP_PATTERN = Pattern.compile("\\R"); 154 return LINE_SEP_PATTERN.split(streamToString(stream)); 155 } 156 157 public static String streamToString(final InputStream input) throws IOException { 158 return new String(input.readAllBytes(), StandardCharsets.UTF_8).replace("\uFEFF", ""); 159 } 160 161 public static byte[] streamToBytes(final InputStream input) throws IOException { 162 if (input == null) { 163 return null; 164 } 165 final byte[] read = input.readAllBytes(); 166 input.close(); 167 return read; 168 } 169 170 public static byte[] streamToBytesNoClose(final InputStream input) throws IOException { 171 if (input == null) { 172 return null; 173 } 174 return input.readAllBytes(); 175 } 176 177 public static void streamToFile(final InputStream stream, final String filename) throws IOException { 178 Files.copy(stream, Path.of(filename), StandardCopyOption.REPLACE_EXISTING); 179 stream.close(); 180 } 181 182 public static void streamToFileNoClose(final InputStream stream, final String filename) throws IOException { 183 Files.copy(stream, Path.of(filename), StandardCopyOption.REPLACE_EXISTING); 184 } 185 186 public static void linesToFile(File f, String[] lines) throws IOException { 187 Files.write(f.toPath(), List.of(lines), StandardCharsets.UTF_8); 188 } 189 190 public static void linesToFile(String path, String[] lines) throws IOException { 191 final File file = ManagedFileAccess.csfile(path); 192 Files.write(file.toPath(), List.of(lines), StandardCharsets.UTF_8); 193 } 194 195 public static void linesToFile(String path, List<String> lines) throws IOException { 196 final File file = ManagedFileAccess.csfile(path); 197 Files.write(file.toPath(), lines, StandardCharsets.UTF_8); 198 } 199 200 public static String fileTitle(String file) throws IOException { 201 if (file == null) 202 return null; 203 String s = ManagedFileAccess.file(file).getName(); 204 return s.indexOf(".") == -1 ? s : s.substring(0, s.indexOf(".")); 205 } 206 207 public static String getDirectoryForFile(String filepath) throws IOException { 208 File f = ManagedFileAccess.file(filepath); 209 return f.getParent(); 210 } 211 212 public static String getDirectoryForFile(File file) throws IOException { 213 return file.getParent(); 214 } 215 216 public static String getRelativePath(String root, String path) { 217 String res = path.substring(root.length()); 218 if (res.startsWith(File.separator)) { 219 res = res.substring(1); 220 } 221 return res; 222 } 223 224 public static File createDirectory(String path) throws IOException { 225 ManagedFileAccess.csfile(path).mkdirs(); 226 return ManagedFileAccess.file(path); 227 } 228 229 public static File createDirectoryNC(String path) throws IOException { 230 ManagedFileAccess.file(path).mkdirs(); 231 return ManagedFileAccess.file(path); 232 } 233 234 public static void copyFile(File sourceFile, File destFile) throws IOException { 235 if (!destFile.exists()) { 236 if (!ManagedFileAccess.csfile(destFile.getParent()).exists()) { 237 createDirectory(destFile.getParent()); 238 } 239 destFile.createNewFile(); 240 } else if (!destFile.getCanonicalFile().getName().equals(destFile.getName())) { 241 // case mismatch 242 destFile.delete(); 243 destFile.createNewFile(); 244 } 245 246 FileInputStream source = null; 247 FileOutputStream destination = null; 248 249 try { 250 source = ManagedFileAccess.inStream(sourceFile); 251 destination = ManagedFileAccess.outStream(destFile); 252 destination.getChannel().transferFrom(source.getChannel(), 0, source.getChannel().size()); 253 } finally { 254 if (source != null) { 255 source.close(); 256 } 257 if (destination != null) { 258 destination.close(); 259 } 260 } 261 } 262 263 public static void copyFile(String source, String dest) throws IOException { 264 copyFile(ManagedFileAccess.file(source), ManagedFileAccess.file(dest)); 265 } 266 267 public static void copyDirectory(String sourceFolder, String destFolder, FileNotifier notifier) throws IOException, FHIRException { 268 CSFile src = ManagedFileAccess.csfile(sourceFolder); 269 if (!src.exists()) 270 throw new FHIRException("Folder " + sourceFolder + " not found"); 271 File dst = ManagedFileAccess.file(destFolder); 272 if(!dst.getCanonicalFile().getName().equals(dst.getName())) { 273 File tmp = ManagedFileAccess.file(destFolder+System.currentTimeMillis()); 274 if (!dst.renameTo(tmp)) { 275 throw new IOException("fixing case from "+dst.getCanonicalFile().getName()+" to "+tmp.getName()+" failed"); 276 } 277 if (!tmp.renameTo(dst)) { 278 throw new IOException("fixing case from "+tmp.getCanonicalFile().getName()+" to "+dst.getName()+" failed"); 279 } 280 } else if (!dst.exists()) { 281 createDirectory(destFolder); 282 } 283 284 String[] files = src.list(); 285 for (String f : files) { 286 if (ManagedFileAccess.csfile(sourceFolder + File.separator + f).isDirectory()) { 287 if (!f.startsWith(".")) { // ignore .git files... 288 copyDirectory(sourceFolder + File.separator + f, destFolder + File.separator + f, notifier); 289 } 290 } else { 291 if (notifier != null) 292 notifier.copyFile(sourceFolder + File.separator + f, destFolder + File.separator + f); 293 FileUtilities.copyFile(ManagedFileAccess.csfile(sourceFolder + File.separator + f), new /*CS*/File(destFolder + File.separator + f)); // case doesn't have to match on the target 294 } 295 } 296 } 297 298 public static void copyDirectory2(String sourceFolder, String destFolder, FileNotifier2 notifier) throws IOException, FHIRException { 299 CSFile src = ManagedFileAccess.csfile(sourceFolder); 300 if (!src.exists()) 301 throw new FHIRException("Folder " + sourceFolder + " not found"); 302 createDirectory(destFolder); 303 304 String[] files = src.list(); 305 for (String f : files) { 306 if (ManagedFileAccess.csfile(sourceFolder + File.separator + f).isDirectory()) { 307 if (!f.startsWith(".")) { // ignore .git files... 308 boolean doCopy = notifier != null ? notifier.copyFolder(sourceFolder + File.separator + f, destFolder + File.separator + f) : true; 309 if (doCopy) { 310 copyDirectory2(sourceFolder + File.separator + f, destFolder + File.separator + f, notifier); 311 } 312 } 313 } else { 314 boolean doCopy = notifier != null ? notifier.copyFile(sourceFolder + File.separator + f, destFolder + File.separator + f) : true; 315 if (doCopy) { 316 FileUtilities.copyFile(ManagedFileAccess.csfile(sourceFolder + File.separator + f), ManagedFileAccess.csfile(destFolder + File.separator + f)); 317 } 318 } 319 } 320 } 321 322 public static void copyFileToDirectory(File source, File destDir) throws IOException { 323 FileUtilities.copyFile(source, ManagedFileAccess.file(Utilities.path(destDir.getAbsolutePath(), source.getName()))); 324 } 325 326 public static void renameDirectory(String source, String dest) throws FHIRException, IOException { 327 File src = ManagedFileAccess.file(source); 328 File dst = ManagedFileAccess.file(dest); 329 if (!src.renameTo(dst)) { 330 int i = 0; 331 do { 332 try { 333 Thread.sleep(20); 334 } catch (Exception e) { 335 // nothing 336 } 337 System.gc(); 338 i++; 339 } while (!src.renameTo(dst) && i < 10); 340 if (src.exists()) { 341 copyDirectory(source, dest, null); 342 try { 343 src.delete(); 344 } catch (Exception e) { 345 // nothing 346 } 347 } 348 } 349 350 } 351 352 public static List<String> listAllFiles(String path, List<String> ignoreList) throws IOException { 353 List<String> res = new ArrayList<>(); 354 scanForFiles(res, path, ManagedFileAccess.file(path), ignoreList); 355 return res; 356 } 357 358 private static void scanForFiles(List<String> res, String root, File dir, List<String> ignoreList) { 359 for (File f : dir.listFiles()) { 360 if (ignoreList == null || !ignoreList.contains(f.getAbsolutePath())) { 361 if (f.isDirectory()) { 362 scanForFiles(res, root, f, ignoreList); 363 } else if (!f.getName().equals(".DS_Store")) { 364 res.add(getRelativePath(root, f.getAbsolutePath())); 365 } 366 } 367 } 368 } 369 370 /** 371 * Delete a directory atomically by first renaming it to a temp directory in 372 * its parent, and then deleting its contents. 373 * 374 * @param path The directory to delete. 375 */ 376 public static void atomicDeleteDirectory(String path) throws IOException { 377 378 File directory = ManagedFileAccess.file(path); 379 380 String tempDirectoryPath = Utilities.generateUniqueRandomUUIDPath(directory.getParent()); 381 File tempDirectory = ManagedFileAccess.file(tempDirectoryPath); 382 if (!directory.renameTo(tempDirectory)) { 383 throw new IOException("Unable to rename directory " + path + " to " + tempDirectory +" for atomic delete"); 384 } 385 clearDirectory(tempDirectory.getAbsolutePath()); 386 if (!tempDirectory.delete()) { 387 throw new IOException("Unable to delete temp directory " + tempDirectory + " when atomically deleting " + path); 388 } 389 } 390 391 public interface FileVisitor { 392 void visitFile(File file) throws FileNotFoundException, IOException; 393 } 394 395 public static void visitFiles(String folder, String extension, FileVisitor visitor) throws FileNotFoundException, IOException { 396 visitFiles(ManagedFileAccess.file(folder), extension, visitor); 397 } 398 399 public static void visitFiles(File folder, String extension, FileVisitor visitor) throws FileNotFoundException, IOException { 400 for (File file : folder.listFiles()) { 401 if (file.isDirectory()) 402 visitFiles(file, extension, visitor); 403 else if (extension == null || file.getName().endsWith(extension)) 404 visitor.visitFile(file); 405 } 406 } 407 408 public static int countFilesInDirectory(String dirName) throws IOException { 409 File dir = ManagedFileAccess.file(dirName); 410 if (dir.exists() == false) { 411 return 0; 412 } 413 int i = 0; 414 for (File f : dir.listFiles()) 415 if (!f.isDirectory()) 416 i++; 417 return i; 418 } 419 420 public static boolean checkFileExists(String purpose, String dir, String file, List<String> errors) 421 throws IOException { 422 if (!ManagedFileAccess.csfile(dir + file).exists()) { 423 if (errors != null) 424 errors.add("Unable to find " + purpose + " file " + file + " in " + dir); 425 return false; 426 } else { 427 return true; 428 } 429 } 430 431 public static boolean checkFolderExists(String dir, List<String> errors) 432 throws IOException { 433 if (!ManagedFileAccess.csfile(dir).exists()) { 434 errors.add("Unable to find directory " + dir); 435 return false; 436 } else { 437 return true; 438 } 439 } 440 441 public static File createTempFile(String prefix, String suffix) throws IOException { 442 // this allows use to eaily identify all our dtemp files and delete them, since delete on Exit doesn't really work. 443 File file = File.createTempFile("ohfu-" + prefix, suffix); 444 file.deleteOnExit(); 445 return file; 446 } 447 448 public static void deleteTempFiles() throws IOException { 449 File file = createTempFile("test", "test"); 450 String folder = getDirectoryForFile(file.getAbsolutePath()); 451 String[] list = ManagedFileAccess.file(folder).list(new FilenameFilter() { 452 public boolean accept(File dir, String name) { 453 return name.startsWith("ohfu-"); 454 } 455 }); 456 if (list != null) { 457 for (String n : list) { 458 ManagedFileAccess.file(Utilities.path(folder, n)).delete(); 459 } 460 } 461 } 462 463 public static void clearDirectory(String folder, String... exemptions) throws IOException { 464 File dir = ManagedFileAccess.file(folder); 465 if (dir.exists()) { 466 if (exemptions.length == 0) 467 FileUtils.cleanDirectory(dir); 468 else { 469 String[] files = ManagedFileAccess.csfile(folder).list(); 470 if (files != null) { 471 for (String f : files) { 472 if (!Utilities.existsInList(f, exemptions)) { 473 File fh = ManagedFileAccess.csfile(folder + File.separatorChar + f); 474 if (fh.isDirectory()) 475 clearDirectory(fh.getAbsolutePath()); 476 fh.delete(); 477 } 478 } 479 } 480 } 481 } 482 } 483 484 public static void deleteAllFiles(String folder, String type) throws IOException { 485 File src = ManagedFileAccess.file(folder); 486 String[] files = src.list(); 487 for (String f : files) { 488 if (ManagedFileAccess.file(folder + File.separator + f).isDirectory()) { 489 deleteAllFiles(folder + File.separator + f, type); 490 } else if (f.endsWith(type)) { 491 ManagedFileAccess.file(folder + File.separator + f).delete(); 492 } 493 } 494 495 } 496 497 public static String changeFileExt(String name, String ext) { 498 if (name.lastIndexOf('.') > -1) 499 return name.substring(0, name.lastIndexOf('.')) + ext; 500 else 501 return name + ext; 502 } 503 504 public static boolean isIgnorableFile(File file) { 505 return Utilities.existsInList(file.getName(), ".DS_Store"); 506 } 507 508 public static void deleteEmptyFolders(File df) { 509 for (File f : df.listFiles()) { 510 if (f.isDirectory()) { 511 deleteEmptyFolders(f); 512 } 513 } 514 boolean empty = true; 515 for (File f : df.listFiles()) { 516 empty = false; 517 break; 518 } 519 if (empty) { 520 df.delete(); 521 } 522 } 523 524 public static String cleanFileName(String badFileName) { 525 StringBuilder cleanName = new StringBuilder(); 526 int len = badFileName.codePointCount(0, badFileName.length()); 527 for (int i=0; i<len; i++) { 528 int c = badFileName.codePointAt(i); 529 if (Arrays.binarySearch(Utilities.illegalChars, c) < 0) { 530 cleanName.appendCodePoint(c); 531 } 532 } 533 return cleanName.toString(); 534 } 535 536 public static void copyFiles(String source, String dest, String... extensions) throws IOException { 537 for (File f : new File(source).listFiles()) { 538 boolean copy = false; 539 for (String e : extensions) { 540 if (f.getName().endsWith(e)) { 541 copy = true; 542 } 543 } 544 if (copy) { 545 FileUtilities.copyFile(f.getAbsolutePath(), Utilities.path(dest, f.getName())); 546 } 547 } 548 549 } 550 551 public static boolean isEmptyDirectory(File dst) { 552 File[] files = dst.listFiles(); 553 return files == null || files.length == 0; 554 } 555 556}