feat: uv python env + DREDge paramset seeds

.gitignore

@@ -8,4 +8,7 @@
 
 Recording_Automation_GUI/
 
-*app_data.mat
+*app_data.mat
+
+# uv-managed virtualenv (uv.lock is committed for reproducibility)
+.venv/

@RecordingProcessJobGUI/getPythonEnv.m

@@ -1,49 +1,157 @@
 function getPythonEnv(app)
-%get python environments filepaths
+%getPythonEnv Resolve how the app's python helper scripts are invoked.
+%
+%   app.py_env is a *command prefix*, not an interpreter path. It runs the
+%   helper scripts through uv, which resolves/syncs the environment declared in
+%   the repo's pyproject.toml on demand:
+%
+%       "<uv>" run --project "<repo_root>" python
+%
+%   Consumers (fillParams, writeParametersDB) just interpolate it into a
+%   system() call, so a multi-word prefix works the same as a bare path.
+%
+%   app.py_ibl_env still points at the external IBL conda env, which this repo
+%   does not own.
+
+repo_root = fileparts(RecordingProcessJobGUI.gui_path);
+
+uv_exe = findUv();
+if isempty(uv_exe)
+    uv_exe = installUv();
+end
+
+if isempty(uv_exe)
+    app.py_env     = [];
+    app.py_enabled = false;
+    warning('RecordingProcessJobGUI:noUv', ...
+        ['Could not find or install uv.\n' ...
+         'Install it manually from https://docs.astral.sh/uv/ and restart the app.']);
+else
+    app.py_env     = ['"' uv_exe '" run --project "' repo_root '" python'];
+    app.py_enabled = true;
+end
+
+% The IBL atlas GUI still lives in a separate conda environment.
+app.py_ibl_env = getCondaEnvPython(RecordingProcessJobGUI.py_iblenv_name);
+
+end
+
+
+function uv_exe = findUv()
+%findUv Locate the uv executable, checking PATH then the standard install dirs.
+
+uv_exe = '';
+
+if ispc
+    exe_name = 'uv.exe';
+    which_cmd = 'where uv';
+else
+    exe_name = 'uv';
+    which_cmd = 'command -v uv';
+end
+
+% 1. Already on PATH?
+[status, out] = system(which_cmd);
+if status == 0
+    lines = strsplit(strtrim(out), newline);
+    candidate = strtrim(lines{1});
+    if ~isempty(candidate) && isfile(candidate)
+        uv_exe = candidate;
+        return
+    end
+end
+
+% 2. Standard install locations. MATLAB's system() inherits a minimal PATH that
+%    typically omits ~/.local/bin and Homebrew, so check them explicitly.
+home = char(java.lang.System.getProperty('user.home'));
+if ispc
+    candidates = { ...
+        fullfile(home, '.local', 'bin', exe_name), ...
+        fullfile(home, 'AppData', 'Local', 'Programs', 'uv', exe_name), ...
+        fullfile(home, 'AppData', 'Roaming', 'uv', 'bin', exe_name)};
+else
+    candidates = { ...
+        fullfile(home, '.local', 'bin', exe_name), ...
+        fullfile('/opt', 'homebrew', 'bin', exe_name), ...
+        fullfile('/usr', 'local', 'bin', exe_name), ...
+        fullfile(home, '.cargo', 'bin', exe_name)};
+end
+
+for i = 1:numel(candidates)
+    if isfile(candidates{i})
+        uv_exe = candidates{i};
+        return
+    end
+end
+
+end
+
+
+function uv_exe = installUv()
+%installUv Bootstrap uv using Astral's official installer, then re-locate it.
+
+uv_exe = '';
+
+if ispc
+    install_cmd = ['powershell -ExecutionPolicy ByPass ' ...
+                   '-c "irm https://astral.sh/uv/install.ps1 | iex"'];
+else
+    install_cmd = 'curl -LsSf https://astral.sh/uv/install.sh | sh';
+end
+
+fprintf('uv not found. Installing it from https://astral.sh/uv ...\n');
+[status, out] = system(install_cmd);
+if status ~= 0
+    fprintf(2, 'uv install failed:\n%s\n', out);
+    return
+end
+
+uv_exe = findUv();
+
+end
+
+
+function py_path = getCondaEnvPython(env_name)
+%getCondaEnvPython Look up a conda env's interpreter by name.
+%
+%   Matches env_name against the leading Name column of `conda env list`, rather
+%   than assuming the name occurs exactly twice in the raw output (which breaks
+%   on names that are prefixes of others, e.g. iblenv vs iblenv2).
+
+py_path = [];
 
 try
-    if ismac
-        this_path = getenv('PATH');
-        this_path = [this_path ':/Users/alvaros/opt/anaconda3/condabin'];
-        setenv('PATH', this_path);
+    [status, conda_envs] = system('conda env list');
+    if status ~= 0
+        return
     end
 
-        % Get all conda environments and find with specific names
-        % (py_env_name, py_iblenv_name)
-        [~, conda_envs] = system('conda env list');
+    lines = strsplit(conda_envs, newline);
+    for i = 1:numel(lines)
+        this_line = strtrim(lines{i});
+        if isempty(this_line) || startsWith(this_line, '#')
+            continue
+        end
 
-        idx_py_env = strfind(conda_envs, RecordingProcessJobGUI.py_env_name);
-
-        app.py_env = strtrim(conda_envs(idx_py_env(1)+length(RecordingProcessJobGUI.py_env_name): ...
-            idx_py_env(2)+length(RecordingProcessJobGUI.py_env_name)));
-
-        if ispc
-            app.py_env = fullfile( app.py_env, 'python');
-            %Surround with double quotes for filepaths with spaces
-            %app.py_env = ['"' app.py_env '"'];
-        else
-            app.py_env = fullfile( app.py_env, 'bin', 'python');
+        tokens = strsplit(this_line);
+        tokens = tokens(~cellfun(@isempty, tokens));
+        if numel(tokens) < 2 || ~strcmp(tokens{1}, env_name)
+            continue
         end
-
-        idx_py_iblenv = strfind(conda_envs, RecordingProcessJobGUI.py_iblenv_name);
-
-        app.py_ibl_env = strtrim(conda_envs(idx_py_iblenv(1)+length(RecordingProcessJobGUI.py_env_name): ...
-            idx_py_iblenv(2)+length(RecordingProcessJobGUI.py_iblenv_name)));
 
+        env_dir = tokens{end};
         if ispc
-            app.py_ibl_env = fullfile( app.py_ibl_env, 'python');
+            candidate = fullfile(env_dir, 'python.exe');
         else
-            app.py_ibl_env = fullfile( app.py_ibl_env, 'bin', 'python');
+            candidate = fullfile(env_dir, 'bin', 'python');
+        end
+        if isfile(candidate)
+            py_path = ['"' candidate '"'];
         end
-
-        %Surround with double quotes for filepaths with spaces
-        app.py_ibl_env = ['"' app.py_ibl_env '"'];
-
-
-
-        app.py_enabled = true;
+        return
+    end
 catch
-    app.py_env = [];
-    app.py_enabled = false;
+    py_path = [];
+end
 
 end

Original_Params_DB/preprocess_paramset_dredge.json

@@ -0,0 +1,10 @@
+{
+    "preset": "dredge",
+    "device": "cuda",
+    "motion_kwargs": {},
+    "job_kwargs": {
+        "n_jobs": -1,
+        "chunk_duration": "1s",
+        "progress_bar": true
+    }
+}

Original_Params_DB/process_paramset_kilosort4_no_drift.json

@@ -0,0 +1,6 @@
+{
+    "n_chan_bin": 385,
+    "nblocks": 0,
+    "do_correction": false,
+    "clustering_method": "kilosort4"
+}

PythonScripts/read_params.py

@@ -5,6 +5,24 @@
 this_dir = os.path.dirname(__file__)
 os.chdir(this_dir)
 
+
+def matlab_safe(obj):
+    """Replace None with [] so scipy.io.savemat can serialize the structure.
+
+    savemat has no encoding for None and raises TypeError on it. Nullable
+    columns and nulls nested inside `params` blobs (e.g. the suite2p imaging
+    paramset) otherwise break the whole .mat file. MATLAB reads [] as empty.
+    """
+    if obj is None:
+        return []
+    if isinstance(obj, dict):
+        return {k: matlab_safe(v) for k, v in obj.items()}
+    if isinstance(obj, list):
+        return [matlab_safe(v) for v in obj]
+    if isinstance(obj, tuple):
+        return tuple(matlab_safe(v) for v in obj)
+    return obj
+
 import datajoint as dj
 dj.conn()
 
@@ -30,13 +48,13 @@
 params_dict_dict = {}
 num_params = 0
 for idx, param_modality_list in enumerate(params_dict_list):
-    for dict in param_modality_list:
-        dict['recording_modality'] = modalities[idx]
-        dict['param_set_hash'] = str(dict['param_set_hash'])
-        if 'clustering_method' in dict:
-            dict['processing_method'] = dict.pop('clustering_method')
-    
-        params_dict_dict['param_'+str(num_params)] = dict
+    for param_dict in param_modality_list:
+        param_dict['recording_modality'] = modalities[idx]
+        param_dict['param_set_hash'] = str(param_dict['param_set_hash'])
+        if 'clustering_method' in param_dict:
+            param_dict['processing_method'] = param_dict.pop('clustering_method')
+
+        params_dict_dict['param_'+str(num_params)] = param_dict
         num_params +=1
 
 #################################################Fetch all preparamsStepList from all modalities
@@ -48,19 +66,19 @@
 preparams_steps_dict_dict = {}
 num_preparams_steps = 0
 for idx, preparam_modality_list in enumerate(preparams_steps):
-    for dict in preparam_modality_list:
-        dict['param_set_hash'] = str(dict['param_set_hash'])
-        dict['recording_modality'] = modalities[idx]
-        if 'precluster_param_steps_id' in dict:
-            dict['preprocess_param_steps_id'] = dict.pop('precluster_param_steps_id')
-        if 'precluster_method' in dict:
-            dict['preprocess_method'] = dict.pop('precluster_method')
-        if 'precluster_param_steps_name' in dict:
-            dict['preprocess_param_steps_name'] = dict.pop('precluster_param_steps_name')
-        if 'precluster_param_steps_desc' in dict:
-            dict['preprocess_param_steps_desc'] = dict.pop('precluster_param_steps_desc')
-
-        preparams_steps_dict_dict['param_'+str(num_preparams_steps)] = dict
+    for param_dict in preparam_modality_list:
+        param_dict['param_set_hash'] = str(param_dict['param_set_hash'])
+        param_dict['recording_modality'] = modalities[idx]
+        if 'precluster_param_steps_id' in param_dict:
+            param_dict['preprocess_param_steps_id'] = param_dict.pop('precluster_param_steps_id')
+        if 'precluster_method' in param_dict:
+            param_dict['preprocess_method'] = param_dict.pop('precluster_method')
+        if 'precluster_param_steps_name' in param_dict:
+            param_dict['preprocess_param_steps_name'] = param_dict.pop('precluster_param_steps_name')
+        if 'precluster_param_steps_desc' in param_dict:
+            param_dict['preprocess_param_steps_desc'] = param_dict.pop('precluster_param_steps_desc')
+
+        preparams_steps_dict_dict['param_'+str(num_preparams_steps)] = param_dict
         num_preparams_steps +=1
 
 #################################################Fetch all preparams from all modalities
@@ -73,13 +91,13 @@
 preparams_dict_dict = {}
 num_preparams = 0
 for idx, preparam_modality_list in enumerate(preparams_dict_list):
-    for dict in preparam_modality_list:
-        dict['recording_modality'] = modalities[idx]
-        dict['param_set_hash'] = str(dict['param_set_hash'])
-        if 'precluster_method' in dict:
-            dict['preprocess_method'] = dict.pop('precluster_method')
-    
-        preparams_dict_dict['param_'+str(num_preparams)] = dict
+    for param_dict in preparam_modality_list:
+        param_dict['recording_modality'] = modalities[idx]
+        param_dict['param_set_hash'] = str(param_dict['param_set_hash'])
+        if 'precluster_method' in param_dict:
+            param_dict['preprocess_method'] = param_dict.pop('precluster_method')
+
+        preparams_dict_dict['param_'+str(num_preparams)] = param_dict
         num_preparams +=1
 
 '''
@@ -119,9 +137,9 @@
 dj.conn().close()
 
 
-savemat('params.mat', params_dict_dict)
-savemat('preparams.mat', preparams_dict_dict)
-savemat('preparams_list.mat', preparams_steps_dict_dict)
+savemat('params.mat', matlab_safe(params_dict_dict))
+savemat('preparams.mat', matlab_safe(preparams_dict_dict))
+savemat('preparams_list.mat', matlab_safe(preparams_steps_dict_dict))
 
 #savemat('methods.mat', methods_dict)
 #savemat('premethods.mat', premethods_dict)

pyproject.toml

@@ -0,0 +1,16 @@
+[project]
+name = "recording-process-job-gui"
+version = "0.1.0"
+description = "Python helper scripts for the RecordingProcessJobGUI MATLAB app"
+requires-python = ">=3.14"
+dependencies = [
+    # datajoint 2.x renamed the config file to datajoint.json and ignores the
+    # dj_local_conf.json this repo ships, so it cannot connect. Stay on 0.14.x.
+    "datajoint<2.0",
+    "element-interface",
+    "pandas",
+    "scipy",
+]
+
+[tool.uv]
+package = false