boot
boot copied to clipboard
Published
20 hours ago •
boot-clj
boot-clj
Creating an uberjar is slow [4min 32s] compared to lein [45s]
Boot Bug Report
Platform details
Platform (macOS, Linux, Windows): macOS
Platform version: 10.12.6
JRE/JDK version (java -version):
java version "1.8.0_121"
Java(TM) SE Runtime Environment (build 1.8.0_121-b13)
Java HotSpot(TM) 64-Bit Server VM (build 25.121-b13, mixed mode)
Boot details
Boot version (2.7.1): 2.7.1
build.boot present? (yes/no): yes
~/.boot/profile present? (yes/no): no
Task name? (if applicable): boot uberjar
build.boot content (if applicable)
(set-env!
:dependencies '[[adzerk/boot-cljs "2.1.4" :scope "test"]
[adzerk/boot-cljs-repl "0.3.3" :scope "test"]
[buddy "2.0.0"]
[ch.qos.logback/logback-classic "1.2.3"]
[cider/cider-nrepl "0.15.0-SNAPSHOT"]
[clj-time "0.14.0"]
[cljs-ajax "0.7.2"]
[com.h2database/h2 "1.4.196"]
[compojure "1.6.0"]
[conman "0.6.9"]
[cprop "0.1.11"]
[crisptrutski/boot-cljs-test "0.3.2-SNAPSHOT" :scope "test"]
[funcool/struct "1.1.0"]
[luminus-http-kit "0.1.4"]
[luminus-migrations "0.4.2"]
[luminus-nrepl "0.1.4"]
[luminus/ring-ttl-session "0.3.2"]
[markdown-clj "1.0.1"]
[metosin/muuntaja "0.3.2"]
[metosin/ring-http-response "0.9.0"]
[mount "0.1.11"]
[org.clojure/clojure "1.8.0"]
[org.clojure/clojurescript "1.9.946" :scope "provided"]
[org.clojure/java.jdbc "0.7.1"]
[org.clojure/tools.cli "0.3.5"]
[org.clojure/tools.logging "0.4.0"]
[org.webjars.bower/tether "1.4.0"]
[org.webjars/bootstrap "4.0.0-alpha.5"]
[org.webjars/font-awesome "4.7.0"]
[re-frame "0.10.2"]
[reagent "0.7.0"]
[reagent-utils "0.2.1"]
[ring-webjars "0.2.0"]
[ring/ring-core "1.6.2"]
[ring/ring-defaults "0.3.1"]
[secretary "1.2.3"]
[selmer "1.11.1"]]
:source-paths #{"src/cljs" "src/cljc" "src/clj"}
:resource-paths #{"resources"})
(require '[adzerk.boot-cljs :refer [cljs]]
'[adzerk.boot-cljs-repl :refer [cljs-repl]])
(deftask dev
"Enables configuration for a development setup."
[]
(set-env!
:source-paths #(conj % "env/dev/clj" "src/cljs" "src/cljc" "env/dev/cljs")
:resource-paths #(conj % "env/dev/resources")
:dependencies #(concat % '[[prone "1.1.4"]
[ring/ring-mock "0.3.0"]
[ring/ring-devel "1.6.1"]
[pjstadig/humane-test-output "0.8.2"]
[binaryage/devtools "0.9.7"]
[com.cemerick/piggieback "0.2.2"]
[crisptrutski/boot-cljs-test "0.3.2-SNAPSHOT" :scope "test"]
[doo "0.1.8"]
[figwheel-sidecar "0.5.14"]
[org.clojure/clojurescript cljs-version :scope "test"]
[org.clojure/tools.nrepl "0.2.12" :scope "test"]
[pandeiro/boot-http "0.7.6" :scope "test"]
[powerlaces/boot-figreload "0.1.1-SNAPSHOT" :scope "test"]
[weasel "0.7.0" :scope "test"]]))
(task-options! repl {:init-ns 'user})
(require 'pjstadig.humane-test-output)
(let [pja (resolve 'pjstadig.humane-test-output/activate!)]
(pja))
identity)
(deftask testing
"Enables configuration for testing."
[]
(dev)
(set-env! :resource-paths #(conj % "env/test/resources"))
(merge-env! :source-paths ["src/cljc" "src/cljs" "test/cljs"])
identity)
(deftask prod
"Enables configuration for production building."
[]
(merge-env! :source-paths #{"env/prod/clj" "env/prod/cljs"}
:resource-paths #{"env/prod/resources"})
identity)
(deftask start-server
"Runs the project without building class files.
This does not pause execution. Combine with a wait task or use the \"run\"
task."
[]
(require 'using-boot.core)
(let [m (resolve 'using-boot.core/-main)]
(with-pass-thru _
(m))))
(deftask run
"Starts the server and causes it to wait."
[]
(comp
(start-server)
(wait)))
(require '[clojure.java.io :as io])
(require '[crisptrutski.boot-cljs-test :refer [test-cljs]])
(deftask figwheel
"Runs figwheel and enables reloading."
[]
(dev)
(require '[powerlaces.boot-figreload :refer [reload]])
(let [reload (resolve 'powerlaces.boot-figreload/reload)]
(comp
(reload :client-opts {:debug true})
(cljs-repl)
(cljs)
(start-server)
(wait))))
(deftask run-cljs-tests
"Runs the doo tests for ClojureScript."
[]
(comp
(testing)
(test-cljs :cljs-opts {:output-to "target/test.js", :main "using-boot.doo-runner", :optimizations :whitespace, :pretty-print true})))
(deftask uberjar
"Builds an uberjar of this project that can be run with java -jar"
[]
(comp
(prod)
(aot :namespace #{'using-boot.core})
(cljs :optimizations :advanced)
(uber)
(jar :file "using-boot.jar" :main 'using-boot.core)
(sift :include #{#"using-boot.jar"})
(target)))
Description
Using a new Luminus project as an example, creating an uberjar using boot takes aprox. 6 times as long as the lein equivalent task. Most of the time seems to be spent in boot.filesystem/patch!.
Steps to reproduce
Using lein:
Setup:
$ lein new luminus using-lein +http-kit +cider +re-frame +h2 +auth
$ cd using-lein
$ time lein uberjar
Running:
Compiling using-lein.env
Compiling using-lein.config
Compiling using-lein.core
Compiling using-lein.db.core
Compiling using-lein.handler
Compiling using-lein.layout
Compiling using-lein.middleware
Compiling using-lein.routes.home
Compiling using-lein.validation
Compiling ClojureScript...
Compiling "target/cljsbuild/public/js/app.js" from ["src/cljc" "src/cljs" "env/prod/cljs"]...
WARNING: uri? already refers to: cljs.core/uri? being replaced by: cognitect.transit/uri? at line 332 /Users/adammoore/src/jartest/using-lein/target/uberjar/cljsbuild-compiler-0/cognitect/transit.cljs
WARNING: Use of undeclared Var cuerdas.core/NaN at line 643 /Users/adammoore/src/jartest/using-lein/target/uberjar/cljsbuild-compiler-0/cuerdas/core.cljc
WARNING: Use of undeclared Var cuerdas.core/NaN at line 646 /Users/adammoore/src/jartest/using-lein/target/uberjar/cljsbuild-compiler-0/cuerdas/core.cljc
Successfully compiled "target/cljsbuild/public/js/app.js" in 15.903 seconds.
Created /Users/adammoore/src/jartest/using-lein/target/uberjar/using-lein-0.1.0-SNAPSHOT.jar
Created /Users/adammoore/src/jartest/using-lein/target/uberjar/using-lein.jar
lein uberjar 131.16s user 9.26s system 307% cpu 45.667 total
Using boot:
Setup:
$ lein new luminus using-boot +http-kit +cider +re-frame +h2 +auth +boot
$ cd using-boot
$ time boot uberjar
Results:
Compiling 1/1 using-boot.core...
Compiling ClojureScript...
• public/js/app.js
WARNING: uri? already refers to: cljs.core/uri? being replaced by: cognitect.transit/uri? at line 332 /Users/adammoore/.boot/cache/tmp/Users/adammoore/src/jartest/using-boot/1mnx/u41uah/public/js/out/cognitect/transit.cljs
Adding uberjar entries...
Writing using-boot.jar...
Sifting output files...
Writing target dir(s)...
boot uberjar 146.24s user 16.51s system 59% cpu 4:32.14 total
Creating a project.clj and using lein is also way faster (note: does not compile cljs):
(deftask lein-uber []
(write-project-clj
:override
{:aot ['using-boot.core]
:main 'using-boot.core
:profiles
{:uberjar {:omit-source true
:aot :all
:uberjar-name "using-boot.jar"
:source-paths #{"src/cljs" "src/cljc" "src/clj" "env/prod/clj"}
:resource-paths #{"resources" "env/prod/resources"}
}}})
(boot.util/dosh "lein" "uberjar"))
boot lein-uber 68.76s user 8.06s system 262% cpu 29.253 total
Hey! Is there any particular step that seemingly takes the majority of the overall time spent?
Seems to be:
https://github.com/boot-clj/boot/blob/master/boot/pod/src/boot/jar.clj#L80
Not sure how to diagnose it fully, sorry! Tried adding some println statements to get a feel of what happens when, but maybe there's a more accurate way.
I have the same issue on an internal project, compiled on WSL with boot 2.7.2. As a workaround, I use a lein-generate task to generate a project.clj file from build.boot and then build the uberjar using lein.
In my case, building the uberjar with lein is about 3 times faster:
$ boot lein-generate
$ time lein uberjar
...
real 1m52.000s user 1m30.828s sys 1m8.000s
$ time boot uberjar
...
real 4m24.905s user 2m1.094s sys 3m20.063s
The resulting file size of the uberjar generated is 42M for lein and 40M for boot.
I hope this approach can help to reproduce this performance issue on other platforms/projects as well. Here is the lein-generate task that I adapted from here to support the creation of a uberjar:
(defn- generate-lein-project-file! [& {:keys [keep-project] :or {keep-project true}}]
(require 'clojure.java.io)
(let [pfile ((resolve 'clojure.java.io/file) "project.clj")
;; Only works when pom options are set using task-options!
{:keys [project version]} (:task-options (meta #'boot.task.built-in/pom))
{:keys [main]} (:task-options (meta #'boot.task.built-in/jar))
prop #(when-let [x (get-env %2)] [%1 x])
head (list* 'defproject (or project 'boot-project) (or version "0.0.0-SNAPSHOT")
(concat
(prop :url :url)
(prop :license :license)
(prop :description :description)
[:dependencies (conj (get-env :dependencies)
['boot/core "2.7.2" :scope "compile"])
:repositories (get-env :repositories)
:source-paths (vec (concat (get-env :source-paths)
(get-env :resource-paths)))]
[:main main
:aot [main]]))
proj (pp-str head)]
(if-not keep-project (.deleteOnExit pfile))
(spit pfile proj)))
(deftask lein-generate
"Generate a leiningen `project.clj` file.
This task generates a leiningen `project.clj` file based on the boot
environment configuration, including project name and version (generated
if not present), dependencies, and source paths. Additional keys may be added
to the generated `project.clj` file by specifying a `:lein` key in the boot
environment whose value is a map of keys-value pairs to add to `project.clj`."
[]
(with-pass-thru fs (generate-lein-project-file! :keep-project true)))
I also switched my project over to using lein uberjar through a util/dosh call until performance improves.
Same problem:
$uname -a
Darwin ====== 20.2.0 Darwin Kernel Version 20.2.0: Wed Dec 2 20:39:59 PST 2020; root:xnu-7195.60.75~1/RELEASE_X86_64 x86_64 i386 MacBookPro15,2 Darwin
$ java -version
java version "1.8.0_251"
Java(TM) SE Runtime Environment (build 1.8.0_251-b08)
Java HotSpot(TM) 64-Bit Server VM (build 25.251-b08, mixed mode)
(vnpy)
$ lein --version
Leiningen 2.9.3 on Java 1.8.0_251 Java HotSpot(TM) 64-Bit Server VM
