feat: Replace pip-compile with uv (#4460)

* Update `shell.nix`

- Replace pip-compile with uv packages
- Pin rust version
- Add var to trigger windmill print more info in stdout

* Replace `pip-compile` with `uv` (dirty + untested)

* Fix arguments passed to uv

Some of the flags are included by default in UV and can be safely removed:
- --resolver=backtracking
- --no-emit-index-url

Also uv does not support `--pip-args` and suggests to directly pass args to uv.

* Remove extra `dbg!`

* Replace 'pip-compile' with 'uv' in Dockerfile

* Add fallback option to `pip-compile` (Disabled)

* Add `uv` to `docker/DockerfileSlim*`

* Add `get_annotation_python` and rename `get_annotation` to `get_annotation_ts`

* Add option to fallback to pip-compile

Put `# no_uv` on top the file for specific python script

Or set `USE_PIP_COMPILE` variable to `true`

* Put back `pip-tools` into shell.nix

* Make sure lockfile resolves again if `#no_uv` used

Add #no_uv to the end of requirements (requirements.in)
That way if something breaks for customer, then they put #no_uv and new lockfile will be resolved

* Put `pip install pip-tools` in original spot

* Fix compilation error

* Fix EE compilation error

error[E0658]: attributes on expressions are experimental
   --> windmill-worker/src/python_executor.rs:144:5
    |
144 |     #[cfg(feature = "enterprise")]
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
    = note: see issue #15701 <rust-lang/rust#15701> for more information

* Add `no_cache` annotation

Will force recalculation of lockfile
And block uv from using cached values

* Target uv cache to /tmp/windmill/cache

* Prohibit uv from managing python

* Add uv to DockerfileBackendTests

* Pin uv version to 0.4.18 in Dockerfiles

* Dont put `#no_uv` in requirements.in

Instead postfix hash for requirements.in with `-no_uv`

* Push Warning to logs if fallbacked to pip-compile

---------

Co-authored-by: Ruben Fiszel <[email protected]>

Author: pyranota

.github/DockerfileBackendTests

@@ -43,7 +43,8 @@ RUN wget https://golang.org/dl/go1.21.5.linux-amd64.tar.gz && tar -C /usr/local
 ENV PATH="${PATH}:/usr/local/go/bin"
 ENV GO_PATH=/usr/local/go/bin/go
 
-RUN curl -LsSf https://astral.sh/uv/install.sh | sh
+# Install UV
+RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh
 
 ENV TZ=Etc/UTC
 

Dockerfile

@@ -158,6 +158,9 @@ RUN set -eux; \
 ENV PATH="${PATH}:/usr/local/go/bin"
 ENV GO_PATH=/usr/local/go/bin/go
 
+# Install UV
+RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh
+
 RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - 
 RUN apt-get -y update && apt-get install -y curl nodejs awscli && apt-get clean \
     && rm -rf /var/lib/apt/lists/*

backend/src/main.rs

@@ -67,7 +67,7 @@ use windmill_worker::{
     get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR,
     BUN_DEPSTAR_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM,
     GO_BIN_CACHE_DIR, GO_CACHE_DIR, LOCK_CACHE_DIR, PIP_CACHE_DIR, POWERSHELL_CACHE_DIR,
-    RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR,
+    RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR, UV_CACHE_DIR,
 };
 
 use crate::monitor::{
@@ -873,6 +873,7 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
         LOCK_CACHE_DIR,
         TMP_LOGS_DIR,
         PIP_CACHE_DIR,
+        UV_CACHE_DIR,
         TAR_PIP_CACHE_DIR,
         DENO_CACHE_DIR,
         DENO_CACHE_DIR_DEPS,

backend/windmill-api/src/jobs.rs

@@ -4127,7 +4127,7 @@ async fn run_dependencies_job(
             JsonRawValue::from_string("true".to_string()).unwrap(),
         );
         if language == ScriptLang::Bun {
-            let annotation = windmill_common::worker::get_annotation(&raw_code);
+            let annotation = windmill_common::worker::get_annotation_ts(&raw_code);
             hm.insert(
                 "npm_mode".to_string(),
                 JsonRawValue::from_string(annotation.npm_mode.to_string()).unwrap(),

backend/windmill-api/src/scripts.rs

@@ -53,7 +53,7 @@ use windmill_common::{
     utils::{
         not_found_if_none, paginate, query_elems_from_hub, require_admin, Pagination, StripPath,
     },
-    worker::{get_annotation, to_raw_value},
+    worker::{get_annotation_ts, to_raw_value},
     HUB_BASE_URL,
 };
 use windmill_git_sync::{handle_deployment_metadata, DeployedObject};
@@ -601,7 +601,7 @@ async fn create_script_internal<'c>(
     };
 
     let lang = if &ns.language == &ScriptLang::Bun || &ns.language == &ScriptLang::Bunnative {
-        let anns = get_annotation(&ns.content);
+        let anns = get_annotation_ts(&ns.content);
         if anns.native_mode {
             ScriptLang::Bunnative
         } else {

backend/windmill-common/src/worker.rs

@@ -303,14 +303,14 @@ fn parse_file<T: FromStr>(path: &str) -> Option<T> {
         .flatten()
 }
 
-pub struct Annotations {
+pub struct TypeScriptAnnotations {
     pub npm_mode: bool,
     pub nodejs_mode: bool,
     pub native_mode: bool,
     pub nobundling: bool,
 }
 
-pub fn get_annotation(inner_content: &str) -> Annotations {
+pub fn get_annotation_ts(inner_content: &str) -> TypeScriptAnnotations {
     let annotations = inner_content
         .lines()
         .take_while(|x| x.starts_with("//"))
@@ -324,7 +324,25 @@ pub fn get_annotation(inner_content: &str) -> Annotations {
     let nobundling: bool =
         annotations.contains(&"nobundling".to_string()) || nodejs_mode || *DISABLE_BUNDLING;
 
-    Annotations { npm_mode, nodejs_mode, native_mode, nobundling }
+    TypeScriptAnnotations { npm_mode, nodejs_mode, native_mode, nobundling }
+}
+
+pub struct PythonAnnotations {
+    pub no_uv: bool,
+    pub no_cache: bool,
+}
+
+pub fn get_annotation_python(inner_content: &str) -> PythonAnnotations {
+    let annotations = inner_content
+        .lines()
+        .take_while(|x| x.starts_with("#"))
+        .map(|x| x.to_string().replace("#", "").trim().to_string())
+        .collect_vec();
+
+    let no_uv: bool = annotations.contains(&"no_uv".to_string());
+    let no_cache: bool = annotations.contains(&"no_cache".to_string());
+
+    PythonAnnotations { no_uv, no_cache }
 }
 
 pub struct SqlAnnotations {

backend/windmill-worker/src/ansible_executor.rs

@@ -33,7 +33,7 @@ use crate::{
         OccupancyMetrics,
     },
     handle_child::handle_child,
-    python_executor::{create_dependencies_dir, handle_python_reqs, pip_compile},
+    python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile},
     AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
     TZ_ENV,
 };
@@ -80,7 +80,7 @@ async fn handle_ansible_python_deps(
             if requirements.is_empty() {
                 "".to_string()
             } else {
-                pip_compile(
+                uv_pip_compile(
                     job_id,
                     &requirements,
                     mem_peak,
@@ -90,6 +90,8 @@ async fn handle_ansible_python_deps(
                     worker_name,
                     w_id,
                     &mut Some(occupancy_metrics),
+                    false,
+                    false,
                 )
                 .await
                 .map_err(|e| {

backend/windmill-worker/src/bun_executor.rs

@@ -47,7 +47,7 @@ use windmill_common::{
     get_latest_hash_for_path,
     jobs::{QueuedJob, PREPROCESSOR_FAKE_ENTRYPOINT},
     scripts::ScriptLang,
-    worker::{exists_in_cache, get_annotation, save_cache, write_file},
+    worker::{exists_in_cache, get_annotation_ts, save_cache, write_file},
     DB,
 };
 
@@ -663,7 +663,7 @@ pub async fn prebundle_bun_script(
     if exists_in_cache(&local_path, &remote_path).await {
         return Ok(());
     }
-    let annotation = get_annotation(inner_content);
+    let annotation = get_annotation_ts(inner_content);
     if annotation.nobundling {
         return Ok(());
     }
@@ -800,7 +800,7 @@ pub async fn handle_bun_job(
     new_args: &mut Option<HashMap<String, Box<RawValue>>>,
     occupancy_metrics: &mut OccupancyMetrics,
 ) -> error::Result<Box<RawValue>> {
-    let mut annotation = windmill_common::worker::get_annotation(inner_content);
+    let mut annotation = windmill_common::worker::get_annotation_ts(inner_content);
 
     let (mut has_bundle_cache, cache_logs, local_path, remote_path) =
         if requirements_o.is_some() && !annotation.nobundling && codebase.is_none() {
@@ -1525,7 +1525,7 @@ pub async fn start_worker(
     let common_bun_proc_envs: HashMap<String, String> =
         get_common_bun_proc_envs(Some(&base_internal_url)).await;
 
-    let mut annotation = windmill_common::worker::get_annotation(inner_content);
+    let mut annotation = windmill_common::worker::get_annotation_ts(inner_content);
 
     //TODO: remove this when bun dedicated workers work without issues
     annotation.nodejs_mode = true;