;; Copyright (c) Cognitect, Inc. All rights reserved.

(ns cognitect.s3-libs.file
  (:refer-clojure :exclude [name])
  (:import [java.io File])
  (:require
   [clojure.java.io :as io]
   [clojure.string :as str]))

(set! *warn-on-reflection* true)

(defn abs
  [& args]
  (.getAbsolutePath ^File (apply io/file args)))

(defn file
  "Like io/file but returns a string"
  [& args]
  (.getPath ^File (apply io/file args)))

(defn parent
  [f]
  (-> f io/file .getAbsoluteFile .getParent))

(defn +ext
  [f ext]
  (str f "." ext))

(defn ext?
  [f ext]
  (str/ends-with? f (str "." ext)))

(defn exists?
  [f]
  (.exists (io/file f)))

(defn name
  [f]
  (.getName (io/file f)))

(defn normal?
  [f]
  (.isFile (io/file f)))

(defn dir?
  [f]
  (.isDirectory (io/file f)))

(defn mkdirs
  [f]
  (.mkdirs (io/file f)))

(defn path-seq
  "Returns paths to all files under base/path, relative to base, filtered by optional
predicate filt."
  ([base path] (path-seq base path (constantly true)))
  ([base path filt]
     (let [base (if (empty? base) "." base)
           abs-base (abs base)
           f (io/file base path)]
       (when (exists? f)
         (->> (file-seq f)
              (filter filt)
              (map abs)
              (remove #{abs-base})
              (map #(subs % (inc (count abs-base))))
              seq)))))

;; Copyright (c) Cognitect, Inc. All rights reserved.

(ns cognitect.s3-libs.specs
  (:require
   [cognitect.s3-libs.file :as file]
   [clojure.edn :as edn]
   [clojure.spec.alpha :as s]))

(defn conform!
  [spec x context]
  (if (s/valid? spec x)
    (s/conform spec x)
    (throw (ex-info (str "Invalid data in " context) (s/explain-data spec x)))))

;; location
(s/def ::type keyword?)
(s/def ::path string?)
(s/def ::location (s/keys :req-un [::path ::type]))
(s/def ::locations (s/coll-of ::location))

;; remote
(s/def ::bucket string?)
(s/def ::key string?)
(s/def ::remote (s/keys :req-un [::bucket ::key]))

;; local
(s/def ::home string?)
(s/def ::local (s/keys :req-un [::home]))

;; path to zip in s3
(s/def ::s3-zip ::path)
;; path to local dir in repo
(s/def ::local-dir ::path)
;; path to local zip in work dir
(s/def ::local-zip ::path)
;; correspondence between local and s3
(s/def ::path-mapping (s/keys :req-un [::s3-zip ::local-zip]
                              :opt-un [::local-dir]))

(s/def ::etag string?)
(s/def ::version-id (s/nilable string?))
(s/def ::upload-result (s/keys :req-un [::etag ::version-id]))
(s/def ::upload-results (s/map-of ::s3-zip ::upload-result))

(s/def ::region string?)
(s/def ::creds-profile string?)
(s/def ::client-args (s/keys :opt-un [::region ::creds-profile]))

(defn unform-s3-uri
  [{:keys [bucket key]}]
  (str "s3://" bucket "/" key))

(defn conform-s3-uri
  "Conformer to :bucket :key for an S3 URI"
  [s3-uri]
  (if-let [[_ bucket key] (re-matches #"s3://([^/]+)/(.*)" s3-uri)]
    {:bucket bucket
     :key key}
    (do ::s/invalid)))

(s/def ::s3-uri (s/conformer conform-s3-uri unform-s3-uri))

(defn file-of
  [spec]
  (fn [f]
    (if (and (string? f) (file/exists? f))
      (let [e (edn/read-string (slurp f))]
        (s/conform spec e))
      (do ::s/invalid))))

(s/def ::sync-direction (s/conformer (fn [x] (get {"--s3" :s3
                                                  "--local" :local}
                                                 x
                                                 ::s/invalid))))
(s/def ::locations-edn (s/conformer (file-of (s/coll-of ::location))))

(s/def ::cli-args
  (s/cat :direction ::sync-direction
         :s3-uri ::s3-uri
         :locations ::locations-edn))
;; Copyright (c) Cognitect, Inc. All rights reserved.

(ns cognitect.s3-libs.zip
  (:import
   java.lang.AutoCloseable
   [java.util.zip ZipEntry ZipFile ZipOutputStream])
  (:require
   [clojure.java.io :as io]
   [cognitect.s3-libs.file :as file]
   [clojure.string :as str]))

(set! *warn-on-reflection* true)

(defn zip-path->dir-path
  [zip-path]
  (assert (file/ext? zip-path "zip"))
  (subs zip-path 0 (- (count zip-path) 4)))

(defn zip-path->dir-base
  [zip]
  (-> zip io/file .getParent))

(defn add-entry
  "Add an entry under name that is an io/copy of content."
  [^ZipOutputStream zo ^String name content]
  (.putNextEntry zo (ZipEntry. name))
  (io/copy content zo)
  (.closeEntry zo)
  nil)

(defn add-dir-entry
  "Add a 'directory' entry under name."
  [^ZipOutputStream zo ^String name]
  (.putNextEntry zo (ZipEntry. (str name "/")))
  (.closeEntry zo)
  nil)

(defn add-dir
  "Add dir to zip under name"
  [zo name dir]
  (doseq [path (file/path-seq dir "")]
    (let [zip-path (str name "/" path)
          local-file (io/file dir path)]
      (if (file/dir? local-file)
        (add-dir-entry zo zip-path)
        (add-entry zo zip-path local-file)))))

(defn entry-paths
  "Returns the set of entry paths in zip-file"
  [zip-file]
  (with-open [zip (ZipFile. (io/file zip-file))]
    (into #{} (map #(.getName ^ZipEntry %))(enumeration-seq (.entries zip)))))

(defn edn->str
  [x]
  (binding [*print-length* nil
            *print-level* nil]
    (pr-str x)))

(defn add-edn
  [zo name edn]
  (add-entry zo name (edn->str edn)))

(defn output-stream
  "Create a ZipOutputStram over io/output-stream able s."
  ^AutoCloseable [s]
  (-> s io/output-stream ZipOutputStream.))

(defn zip-dir
  "Compress dir to zip-file. Includes dir name as the first element of
each zip entry. Throws on failure."
  [dir zip-file]
  (with-open [zo (output-stream zip-file)]
    (doseq [path (file/path-seq dir "")]
      (let [zip-path (str (file/name dir) "/" path)
            local-file (io/file dir path)]
        (if (file/dir? local-file)
          (add-dir-entry zo zip-path)
          (add-entry zo zip-path local-file)))))
  {})

(defn ensure-zip-dir
  "Zips dir to zip-path iff zip-path does not exist. Returns :created or
:exists."
  [dir zip-path]
  (if (.exists (io/file zip-path))
    :exists
    (do
      (io/make-parents zip-path)
      (zip-dir dir zip-path)
      :created)))

(defn unzip
  "Unzip zipfile in a dir. Returns nil or throws."
  [zip-path dir]
  (with-open [zip (ZipFile. (io/file zip-path))]
    (doseq [^ZipEntry entry (->> (.entries zip)
                                 (enumeration-seq)
                                 (remove #(.isDirectory ^ZipEntry %)))]
      (let [dest (io/file dir (.getName entry))]
        (.mkdirs (.getParentFile dest))
        (io/copy (.getInputStream zip entry) dest))))
  nil)

(defn ensure-unzip
  "Unzips zip-path to parent of dir iff dir does not exist.
dir defaults to zip path minus the .zip. Returns :created or :exists"
  ([zip-file dir]
     (if (.exists (io/file dir))
       :exists
       (do
         (unzip zip-file (file/parent dir))
         :created))))






;; Copyright (c) Cognitect, Inc. All rights reserved.

;; locations logic with no deps outside Clojure!
(ns cognitect.s3-libs.locations
  (:require
   [clojure.spec.alpha :as s]
   [cognitect.s3-libs.file :as file]
   [cognitect.s3-libs.specs :as specs]
   [cognitect.s3-libs.zip :as zip]))

(def work-dir ".cognitect-s3-libs")

(defmulti repo-base (fn [coord-type] coord-type))
(defmethod repo-base :mvn [_] ".m2/repository")
(defmethod repo-base :git [_] ".gitlibs/libs")

(s/fdef location->s3-zip-rel
        :args (s/cat :location ::specs/location))
(defn location->s3-zip-rel
  "Returns key relative to S3 lib base."
  [{:keys [path type]}]
  (str (name type) "/" path ".zip"))

(s/fdef location->s3-zip
        :args (s/cat :location ::specs/location
                     :key ::specs/key))
(defn location->s3-zip
  "Returns full key in S3."
  [location key]
  (str key "/" (location->s3-zip-rel location)))

(s/fdef location->local-dir
        :args (s/cat :location ::specs/location
                     :home ::specs/home)) 
(defn location->local-dir
  [{:keys [path type]} home]
  (file/file home (repo-base type) path))

(s/fdef location->local-zip
        :args (s/cat :location ::specs/location
                     :home ::specs/home)) 
(defn location->local-zip
  [{:keys [path type]} home]
  (-> (file/file home work-dir (repo-base type) path)
      (file/+ext "zip")))

(s/fdef locations->needed-local
        :args (s/cat :remote ::specs/remote
                     :local ::specs/local)) 
(defn locations->needed-local
  "Returns a transformer from locations to mappings, omitting
any files already present locally"
  [{:keys [key]} {:keys [home]}]
  (let [ld #(location->local-dir % home)
        lz #(location->local-zip % home)]
    (comp
     (remove (comp file/exists? ld))
     (map (fn [location]
            {:s3-zip (location->s3-zip location key)
             :local-zip (lz location)
             :local-dir (ld location)})))))




;; Copyright (c) Cognitect, Inc. All rights reserved.

(ns datomic.ion.dev.ensure-local
  ;; NO DEPS OUTSIDE CLOJURE and S3-LIBS!
  ;; IF YOU ADD TO THIS YOU MAY ALSO NEED TO UPDATE bundle/ensure-local-script!
  (:require
   [clojure.pprint :as pp]
   [clojure.spec.alpha :as s]
   [clojure.string :as str]
   [clojure.java.shell :as sh]
   [cognitect.s3-libs.file :as file]
   [cognitect.s3-libs.locations :as locs]
   [cognitect.s3-libs.specs :as s3-libs-specs]
   [cognitect.s3-libs.zip :as zip]))

(defn cli-include
  [{:keys [path]}]
  ["--include" (file/+ext path "zip")])

(defn download-commands
  [{:keys [bucket key]} {:keys [home]} locations]
  (into
   []
   (comp (map :type)
         (distinct)
         (map (fn [type]
                (into ["aws" "s3" "cp"
                       (str "s3://" bucket "/" key "/" (name type))
                       (file/file home locs/work-dir (locs/repo-base type))
                       "--recursive"
                       "--only-show-errors"
                       ;; not a shell, so no quoting on the *
                       "--exclude" "*"]
                      (mapcat cli-include (filter #(= type (:type %))
                                                  locations))))))
   locations))

(defn sh!
  [& args]
  (let [{:keys [exit] :as result} (apply sh/sh args)]
    (if (zero? exit)
      result
      (throw (ex-info "Shell command failed" {:args args :result result})))))

(defn download
  [remote local locations]
  (doseq [cmd (download-commands remote local locations)]
    (prn (str/join " " cmd))
    (apply sh! cmd)))

(defmulti location-unzipped? (fn [type dir] type))

(defmethod location-unzipped? :mvn
  [_ d]
  (boolean (some #(file/ext? % "jar") (file/path-seq d ""))))

(defmethod location-unzipped? :git
  [_ d]
  (file/exists? d))

(defn location-needed?
  [{:keys [type] :as location} {:keys [home] :as arg2}]
  (let [d (locs/location->local-dir location home)]
    (not (location-unzipped? type d))))

(s/fdef run
        :args (s/cat :remote ::s3-libs-specs/remote
                     :local ::s3-libs-specs/local
                     :locations ::s3-libs-specs/locations))
(defn run
  "Uses s3-uri to ensure that libs are installed locally. Blocking,
,internally concurrent, throws on failure."
  [{:keys [bucket] :as remote} {:keys [home] :as local} locations]
  (let [needed-local (seq (filter #(location-needed? % local) locations))]
    (when (seq needed-local)
      (pp/pprint {:downloading needed-local})
      (download remote local needed-local)
      (doseq [loc needed-local]
        (zip/unzip (locs/location->local-zip loc home)
                   (file/parent (locs/location->local-dir loc home)))))))

(s/def ::args (s/cat :remote ::s3-libs-specs/s3-uri
                     :home string?
                     :locations ::s3-libs-specs/locations-edn))
(defn -main
  [& args]
  (try
   (let [{:keys [remote home locations] :as args} (s3-libs-specs/conform! ::args args ::-main)]
     (pp/pprint {:syncing args})
     (run remote {:home home} locations))
   (finally
    (shutdown-agents))))


