From 0ca482183d7ad06c50b7b5207ef0069e7d0e2414 Mon Sep 17 00:00:00 2001 From: Adam Nowak Date: Fri, 10 Jul 2026 15:01:11 +0200 Subject: [PATCH 1/8] Restructure deps: pip-only pyproject with [embeddings] extra - Drop unused conda-only pdbfixer and the erroneous mamba (conda manager) deps - Move torch/esm/transformers into optional [embeddings] extra (used only by the embedding path, which already imports lazily) - Move pytest/pytest-order into [test]; add [dev] aggregate extra - Replace bs4 with beautifulsoup4; drop redundant bio (biopython kept) - Add PyPI metadata (description, license, authors, classifiers, urls) - Guard embedder lazy imports with a clear 'pip install FRIdata[embeddings]' hint --- pyproject.toml | 51 +++++++++++++++---- .../embedding/embedder/embedder_type.py | 29 +++++++---- 2 files changed, 61 insertions(+), 19 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e4b748a..f7b855a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,8 +5,23 @@ build-backend = "setuptools.build_meta" [project] name = "FRIdata" version = "0.1.0" +description = "Generate sequences, coordinates, distograms, and embeddings from protein structures at scale." readme = "README.md" requires-python = ">=3.10" +license = { text = "BSD-3-Clause" } +authors = [{ name = "Tomasz Lab" }] +keywords = ["bioinformatics", "protein-structure", "dask", "dataset"] +classifiers = [ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + "Intended Audience :: Science/Research", +] + +# Core dependencies. Everything needed for sequences, coordinates and distograms. +# Heavy ML dependencies (torch/esm/transformers) live in the [embeddings] extra so a +# default install stays lightweight — see optional-dependencies below. dependencies = [ "foldcomp", "pydantic>=2.8.2", @@ -16,24 +31,40 @@ dependencies = [ "biotite>=0.41.2", "numpy>=1.26.4,<1.27.0", "genson", - "bs4", - "bio~=1.7.0", - "pytest", - "pytest-order", + "beautifulsoup4", + "biopython", "dask>=2024.4.2,<2025.0.0", "dask[dataframe]", "distributed>=2024.4.2,<2025.0.0", "python-dotenv", "tables", - "biopython", "bokeh>=2.4.2,<3.0.0", - "pdbfixer", - "mamba", - "transformers", "tqdm", +] + +[project.optional-dependencies] +# Embedding generation. Installs torch/esm/transformers from the default PyPI index +# (CUDA-enabled on Linux x86_64). For a specific CUDA version or CPU-only wheels on +# Linux, use scripts/install_pytorch.sh or a manual `--index-url` install afterwards. +embeddings = [ "torch", - "esm" + "esm", + "transformers", ] +test = [ + "pytest", + "pytest-order", +] +dev = [ + "FRIdata[embeddings,test]", + "build", + "twine", +] + +[project.urls] +Homepage = "https://github.com/Tomasz-Lab/FRIdata" +Repository = "https://github.com/Tomasz-Lab/FRIdata" +Issues = "https://github.com/Tomasz-Lab/FRIdata/issues" [project.scripts] fridata = "fridata:main" @@ -45,4 +76,4 @@ include-package-data = true [tool.setuptools.packages.find] where = ["."] -include = ["toolbox*"] \ No newline at end of file +include = ["toolbox*"] diff --git a/toolbox/models/embedding/embedder/embedder_type.py b/toolbox/models/embedding/embedder/embedder_type.py index 383209f..d0bf9eb 100644 --- a/toolbox/models/embedding/embedder/embedder_type.py +++ b/toolbox/models/embedding/embedder/embedder_type.py @@ -7,20 +7,31 @@ from toolbox.models.embedding.embedder.base_embedder import BaseEmbedder +_EMBEDDINGS_EXTRA_HINT = ( + "Embedding generation requires the optional 'embeddings' dependencies " + "(torch, esm, transformers). Install them with:\n" + " pip install 'FRIdata[embeddings]'\n" + "or, for a specific CUDA version / CPU-only build, run scripts/install_pytorch.sh." +) + + def _lazy_embedder_class(member: "EmbedderType") -> Type[BaseEmbedder]: name = member.name - if name.startswith("ESM2"): - from toolbox.models.embedding.embedder.esm2_embedder import ESM2Embedder + try: + if name.startswith("ESM2"): + from toolbox.models.embedding.embedder.esm2_embedder import ESM2Embedder - return ESM2Embedder - if name.startswith("ESMC"): - from toolbox.models.embedding.embedder.esmc_embedder import ESMCEmbedder + return ESM2Embedder + if name.startswith("ESMC"): + from toolbox.models.embedding.embedder.esmc_embedder import ESMCEmbedder - return ESMCEmbedder - if name.startswith("GLM2"): - from toolbox.models.embedding.embedder.glm2_embedder import GLM2Embedder + return ESMCEmbedder + if name.startswith("GLM2"): + from toolbox.models.embedding.embedder.glm2_embedder import GLM2Embedder - return GLM2Embedder + return GLM2Embedder + except ImportError as exc: + raise ImportError(f"{exc}\n\n{_EMBEDDINGS_EXTRA_HINT}") from exc raise ValueError(f"Unknown embedder kind for {name!r}") From 4568278df92460194d800fd02707f8a724a518bc Mon Sep 17 00:00:00 2001 From: Adam Nowak Date: Fri, 10 Jul 2026 15:02:10 +0200 Subject: [PATCH 2/8] Convert setup_env.sh to venv + pip; make install_pytorch.sh interpreter-agnostic - setup_env.sh now creates a python -m venv and pip-installs FRIdata[dev] (editable), then calls install_pytorch.sh for a CUDA-matched torch build - install_pytorch.sh accepts a PYTHON env var to target a specific interpreter and no longer requires an active conda environment --- scripts/install_pytorch.sh | 33 +++++++-------- scripts/setup_env.sh | 84 +++++++++++++++----------------------- 2 files changed, 51 insertions(+), 66 deletions(-) diff --git a/scripts/install_pytorch.sh b/scripts/install_pytorch.sh index 3ae3f32..d1b22f1 100755 --- a/scripts/install_pytorch.sh +++ b/scripts/install_pytorch.sh @@ -15,9 +15,17 @@ # Usage: ./install_pytorch.sh [--cpu] [--dry-run] # --cpu Force the CPU-only PyTorch build (skip GPU detection) # --dry-run Print what would be installed without installing anything +# +# Set the PYTHON environment variable to target a specific interpreter (for +# example a virtualenv's python) without activating it first: +# PYTHON=.venv/bin/python ./install_pytorch.sh set -euo pipefail +# Interpreter to install into. Defaults to whatever 'python' is on PATH so an +# activated venv/conda environment is picked up automatically. +PYTHON="${PYTHON:-python}" + # PyTorch wheel CUDA versions, ascending. Update as new wheels are released. # https://pytorch.org/get-started/locally/ and https://download.pytorch.org/whl/ SUPPORTED_CUDA=(11.8 12.1 12.4 12.6 12.8) @@ -40,21 +48,14 @@ for arg in "$@"; do esac done -# --- Verify an environment is active ------------------------------------------ -if ! command -v python >/dev/null 2>&1; then - echo "Error: no 'python' found on PATH. Activate your conda/venv first." >&2 +# --- Verify the target interpreter exists ------------------------------------- +if ! "$PYTHON" -c 'import sys' >/dev/null 2>&1; then + echo "Error: '$PYTHON' is not a usable Python interpreter." >&2 + echo " Activate a venv first, or set PYTHON=/path/to/venv/bin/python." >&2 exit 1 fi -if [ -n "${CONDA_PREFIX:-}" ]; then - echo "Target environment: $CONDA_PREFIX" - if [ "$(basename "$CONDA_PREFIX")" = "base" ]; then - echo "Warning: the conda 'base' environment is active. It is strongly" >&2 - echo " recommended to activate a dedicated environment first." >&2 - fi -else - echo "Warning: CONDA_PREFIX is not set; installing into '$(command -v python)'." >&2 -fi +echo "Target interpreter: $("$PYTHON" -c 'import sys; print(sys.executable)')" # Turn a X.Y version into a comparable integer, e.g. 12.4 -> 1204. cuda_to_int() { @@ -112,11 +113,11 @@ else INDEX_URL="https://download.pytorch.org/whl/${CUDA_TAG}" fi -PIP_CMD=(python -m pip install torch torchvision torchaudio --index-url "$INDEX_URL") +PIP_CMD=("$PYTHON" -m pip install torch torchvision torchaudio --index-url "$INDEX_URL") echo echo "Will run: ${PIP_CMD[*]}" -echo "Then: python -m pip install esm" +echo "Then: $PYTHON -m pip install esm" if [ "$DRY_RUN" = true ]; then echo "(--dry-run: nothing installed)" @@ -129,11 +130,11 @@ echo "Installing PyTorch..." # ESM requires PyTorch to be installed first. echo "Installing ESM..." -python -m pip install esm +"$PYTHON" -m pip install esm # --- Verify the installation -------------------------------------------------- echo "Verifying PyTorch installation..." -python - <<'PY' +"$PYTHON" - <<'PY' import torch print(f"torch {torch.__version__}") if torch.cuda.is_available(): diff --git a/scripts/setup_env.sh b/scripts/setup_env.sh index f169825..aa4e4a4 100755 --- a/scripts/setup_env.sh +++ b/scripts/setup_env.sh @@ -1,21 +1,21 @@ #!/bin/bash -# Create a FRIdata conda/mamba environment and install Python + PyTorch dependencies. +# Create a FRIdata virtual environment and install its dependencies with pip. # -# Pip packages are installed from requirements-fridata.txt after the conda solve so -# installs work even when this repository checkout is not writable (libmamba writes -# temporary pip requirement files next to the environment YAML). +# This creates a plain Python venv, installs FRIdata (editable) with its dev +# extras, and then installs a PyTorch build matched to your GPU driver via +# install_pytorch.sh. No conda/mamba required. # # Usage: ./scripts/setup_env.sh [options] -# -n, --name NAME Environment name (default: fridata_env) +# -p, --path PATH Virtualenv directory (default: .venv) # --cpu Install CPU-only PyTorch -# --skip-pytorch Skip PyTorch/ESM installation +# --skip-pytorch Skip PyTorch/ESM installation (core install only) # -h, --help Show this help set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -ENV_NAME="fridata_env" +VENV_PATH="$REPO_ROOT/.venv" CPU_ONLY=false SKIP_PYTORCH=false @@ -25,8 +25,8 @@ usage() { while [[ $# -gt 0 ]]; do case "$1" in - -n|--name) - ENV_NAME="$2" + -p|--path) + VENV_PATH="$2" shift 2 ;; --cpu) @@ -49,59 +49,44 @@ while [[ $# -gt 0 ]]; do esac done -if command -v mamba >/dev/null 2>&1; then - MAMBA=mamba -elif command -v micromamba >/dev/null 2>&1; then - MAMBA=micromamba +# Pick a Python interpreter (prefer python3). +if command -v python3 >/dev/null 2>&1; then + PYTHON=python3 +elif command -v python >/dev/null 2>&1; then + PYTHON=python else - echo "Error: mamba or micromamba is required but not found on PATH." >&2 + echo "Error: no python3/python found on PATH." >&2 exit 1 fi -YAML_SRC="$REPO_ROOT/fridata_env_conda.yml" -REQS="$REPO_ROOT/requirements-fridata.txt" -WORKDIR="$(mktemp -d)" -trap 'rm -rf "$WORKDIR"' EXIT +echo "Creating virtual environment at '$VENV_PATH'..." +"$PYTHON" -m venv "$VENV_PATH" -if [ ! -f "$YAML_SRC" ]; then - echo "Error: environment file not found: $YAML_SRC" >&2 - exit 1 -fi - -if [ ! -f "$REQS" ]; then - echo "Error: requirements file not found: $REQS" >&2 - exit 1 +# Use the venv's interpreter directly so this works without 'source activate'. +VENV_PY="$VENV_PATH/bin/python" +if [ ! -x "$VENV_PY" ]; then + # Windows Git Bash layout. + VENV_PY="$VENV_PATH/Scripts/python.exe" fi -# Copy the YAML to a writable directory. This avoids libmamba permission errors when -# the repository checkout itself is read-only and keeps older mamba versions working. -cp "$YAML_SRC" "$WORKDIR/fridata_env_conda.yml" +echo "Upgrading pip..." +"$VENV_PY" -m pip install --upgrade pip -echo "Creating conda environment '$ENV_NAME'..." -if "$MAMBA" env create -f "$WORKDIR/fridata_env_conda.yml" -n "$ENV_NAME" -y; then - : -elif "$MAMBA" env list | awk '{print $1}' | grep -qx "$ENV_NAME"; then - echo "Environment '$ENV_NAME' already exists; updating conda packages..." - "$MAMBA" env update -n "$ENV_NAME" -f "$WORKDIR/fridata_env_conda.yml" -y +if [ "$SKIP_PYTORCH" = true ]; then + echo "Installing FRIdata with dev + test extras (no embeddings/PyTorch)..." + "$VENV_PY" -m pip install -e "$REPO_ROOT[test]" else - echo "Error: failed to create environment '$ENV_NAME'." >&2 - exit 1 -fi + echo "Installing FRIdata with dev extras..." + "$VENV_PY" -m pip install -e "$REPO_ROOT[dev]" -echo "Installing pip requirements from requirements-fridata.txt..." -export TMPDIR="${TMPDIR:-/tmp}" -"$MAMBA" run -n "$ENV_NAME" python -m pip install -r "$REQS" - -if [ "$SKIP_PYTORCH" = false ]; then + # install_pytorch.sh selects a torch wheel matching the machine's CUDA driver + # and reinstalls torch/esm accordingly. It operates on the interpreter it is + # given via the PYTHON env var. PYTORCH_ARGS=() if [ "$CPU_ONLY" = true ]; then PYTORCH_ARGS+=(--cpu) fi - - # install_pytorch.sh expects an activated environment. - eval "$("$MAMBA" shell hook --shell bash)" - mamba activate "$ENV_NAME" - bash "$SCRIPT_DIR/install_pytorch.sh" "${PYTORCH_ARGS[@]}" + PYTHON="$VENV_PY" bash "$SCRIPT_DIR/install_pytorch.sh" "${PYTORCH_ARGS[@]}" fi cat < Date: Fri, 10 Jul 2026 15:03:05 +0200 Subject: [PATCH 3/8] Convert HPC SLURM scripts from conda to venv + pip - initialize_slurm.sh creates a python -m venv, pip-installs FRIdata, and uses install_pytorch.sh for a CUDA-matched torch build (no GROUP_DIR/.conda needed) - common_slurm.sh loads a Python module and activates the venv (VENV_PATH, default $DEEPFRI_PATH/.venv) instead of conda - Module name candidate lists flagged as cluster-specific for easy adjustment --- scripts/hpc/common_slurm.sh | 26 +++++++---- scripts/hpc/initialize_slurm.sh | 83 ++++++++++++++++----------------- 2 files changed, 56 insertions(+), 53 deletions(-) diff --git a/scripts/hpc/common_slurm.sh b/scripts/hpc/common_slurm.sh index 5d83e7f..4f0b09f 100644 --- a/scripts/hpc/common_slurm.sh +++ b/scripts/hpc/common_slurm.sh @@ -21,9 +21,9 @@ start_computation() { cd $DEEPFRI_PATH - # Robustly try to load GCC and a Conda/Miniconda module (handle varied names) + # Robustly try to load GCC and a Python module (handle varied names) LOADED_GCC=false - LOADED_CONDA=false + LOADED_PYTHON=false if command -v module >/dev/null 2>&1; then GCC_CANDIDATES=(gcc GCC) for MOD in "${GCC_CANDIDATES[@]}"; do @@ -34,11 +34,12 @@ start_computation() { fi done - CONDA_CANDIDATES=(miniconda3 Miniconda3 miniconda Anaconda3 anaconda3) - for MOD in "${CONDA_CANDIDATES[@]}"; do + # Cluster-specific: adjust these names to match `module avail python`. + PYTHON_CANDIDATES=(python Python python3 Python3) + for MOD in "${PYTHON_CANDIDATES[@]}"; do if module load "$MOD" >/dev/null 2>&1; then echo "Loaded module: $MOD" - LOADED_CONDA=true + LOADED_PYTHON=true break fi done @@ -49,13 +50,20 @@ start_computation() { exit 1 fi - if [ "$LOADED_CONDA" = false ]; then - echo "Error: Could not load a Conda module." + if [ "$LOADED_PYTHON" = false ]; then + echo "Error: Could not load a Python module." exit 1 fi - eval "$(conda shell.bash hook)" - conda activate $CONDA_ENV_PATH + # Activate the FRIdata virtualenv created by initialize_slurm.sh. + if [[ ! -v VENV_PATH ]]; then + VENV_PATH="$DEEPFRI_PATH/.venv" + fi + if [ ! -f "$VENV_PATH/bin/activate" ]; then + echo "Error: virtualenv not found at '$VENV_PATH'. Run initialize_slurm.sh first." + exit 1 + fi + source "$VENV_PATH/bin/activate" echo "Start time: `date`" start_time=$(date +%s) diff --git a/scripts/hpc/initialize_slurm.sh b/scripts/hpc/initialize_slurm.sh index 39781bb..801547c 100644 --- a/scripts/hpc/initialize_slurm.sh +++ b/scripts/hpc/initialize_slurm.sh @@ -1,16 +1,16 @@ #!/bin/bash -# Installation script for toolbox environment -# Usage: ./install.sh [--cpu] +# Installation script for the FRIdata HPC environment (venv + pip). +# Usage: ./initialize_slurm.sh [--cpu] # --cpu: Install CPU-only PyTorch (for CI or non-GPU systems) -# Default: Install GPU-enabled PyTorch +# Default: Install GPU-enabled PyTorch matched to the node's CUDA driver +# +# Environment variables: +# DEEPFRI_PATH (required) parent directory of the FRIdata repo +# VENV_PATH (optional) virtualenv location, defaults to $DEEPFRI_PATH/.venv set -e -GROUP_DIR=$1 CPU_ONLY=false - -# Parse optional flags -shift for arg in "$@"; do case $arg in --cpu) @@ -19,69 +19,64 @@ for arg in "$@"; do esac done -if [ -z "$GROUP_DIR" ]; then - echo "Usage: ./install.sh [--cpu]" - exit 1 -fi - # Check if DEEPFRI_PATH is set, if not then throw an error if [[ ! -v DEEPFRI_PATH ]]; then echo "Error: DEEPFRI_PATH environment variable is not set." exit 1 fi -# Check if CONDA_ENV_PATH is set, if not then set a default path -if [[ ! -v CONDA_ENV_PATH ]]; then - CONDA_ENV_PATH="$DEEPFRI_PATH/conda_dev" +# Virtualenv location (mirrors common_slurm.sh). +if [[ ! -v VENV_PATH ]]; then + VENV_PATH="$DEEPFRI_PATH/.venv" fi -CONDA_DIR="$GROUP_DIR/.conda" - -# Try loading a Conda/Miniconda module in a robust way (handle varied names) -LOADED_MODULE=false +# Try loading GCC and a Python module in a robust way (handle varied names). +LOADED_GCC=false +LOADED_PYTHON=false if command -v module >/dev/null 2>&1; then - MODULE_CANDIDATES=(miniconda3 Miniconda3 miniconda Anaconda3 anaconda3) - for MOD in "${MODULE_CANDIDATES[@]}"; do + GCC_CANDIDATES=(gcc GCC) + for MOD in "${GCC_CANDIDATES[@]}"; do + if module load "$MOD" >/dev/null 2>&1; then + echo "Loaded module: $MOD" + LOADED_GCC=true + break + fi + done + + # Cluster-specific: adjust these names to match `module avail python`. + PYTHON_CANDIDATES=(python Python python3 Python3) + for MOD in "${PYTHON_CANDIDATES[@]}"; do if module load "$MOD" >/dev/null 2>&1; then echo "Loaded module: $MOD" - LOADED_MODULE=true + LOADED_PYTHON=true break fi done fi -if [ "$LOADED_MODULE" = false ]; then - echo "Error: Could not load a Conda module." +if [ "$LOADED_PYTHON" = false ]; then + echo "Error: Could not load a Python module." exit 1 fi -conda config --add pkgs_dirs "$CONDA_DIR" +# Create the virtualenv. +echo "Creating virtualenv at '$VENV_PATH'..." +python -m venv "$VENV_PATH" +source "$VENV_PATH/bin/activate" -# Create environment from base YAML (without PyTorch or pip-only deps) -ENV_WORKDIR="$(mktemp -d)" -trap 'rm -rf "$ENV_WORKDIR"' EXIT -cp "$DEEPFRI_PATH/FRIdata/fridata_env_conda.yml" "$ENV_WORKDIR/fridata_env_conda.yml" -conda env create --prefix $CONDA_ENV_PATH --file "$ENV_WORKDIR/fridata_env_conda.yml" +python -m pip install --upgrade pip -conda config --set auto_activate_base false +echo "Installing FRIdata core dependencies..." +pip install -e "$DEEPFRI_PATH/FRIdata" -eval "$(conda shell.bash hook)" -conda activate $CONDA_ENV_PATH - -echo "Installing pip requirements..." -pip install -r "$DEEPFRI_PATH/FRIdata/requirements-fridata.txt" - -# Install PyTorch based on mode +# Install PyTorch/ESM matched to the node (GPU by default, CPU with --cpu). +PYTORCH_ARGS=() if [ "$CPU_ONLY" = true ]; then echo "Installing CPU-only PyTorch..." - conda install -y pytorch cpuonly -c pytorch + PYTORCH_ARGS+=(--cpu) else echo "Installing GPU-enabled PyTorch..." - conda install -y pytorch-gpu fi - -# Install ESM (requires PyTorch to be installed first) -echo "Installing ESM..." -pip install esm +PYTHON="$VENV_PATH/bin/python" bash "$DEEPFRI_PATH/FRIdata/scripts/install_pytorch.sh" "${PYTORCH_ARGS[@]}" echo "Installation complete!" From 805b281aaa8c54a3e3331a100abc8d3ea3535eb3 Mon Sep 17 00:00:00 2001 From: Adam Nowak Date: Fri, 10 Jul 2026 15:03:29 +0200 Subject: [PATCH 4/8] Rewrite CI to use setup-python + pip instead of micromamba - actions/setup-python with a 3.10/3.11/3.12 matrix and pip caching - Install CPU torch from the PyTorch index first, then FRIdata[embeddings,test] - Drops the entire micromamba/libmamba caching dance (~60 lines -> ~15) --- .github/workflows/ci.yml | 75 ++++++++-------------------------------- 1 file changed, 14 insertions(+), 61 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d1b6e11..6071c81 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,74 +6,27 @@ on: jobs: test: runs-on: ubuntu-latest - defaults: - run: - shell: bash -el {0} + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12"] steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Generate cache key - id: cache-key - run: | - # Create a hash from the environment files and PyTorch CPU install string - ENV_HASH=$(cat fridata_env_conda.yml requirements-fridata.txt | sha256sum | cut -d' ' -f1) - PYTORCH_SPEC="pip-torch-torchvision-cpu-whl" - CACHE_KEY="micromamba-${{ runner.os }}-${ENV_HASH}-${PYTORCH_SPEC}" - echo "key=${CACHE_KEY}" >> $GITHUB_OUTPUT - - - name: Restore micromamba environment cache - id: cache-restore - uses: actions/cache/restore@v4 - with: - path: | - ~/micromamba/envs - ~/micromamba-bin - key: ${{ steps.cache-key.outputs.key }} - - - name: Setup micromamba (cache miss) - if: steps.cache-restore.outputs.cache-hit != 'true' - uses: mamba-org/setup-micromamba@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 with: - environment-file: fridata_env_conda.yml - environment-name: fridata_env - cache-environment: false - cache-downloads: false - micromamba-binary-path: ~/micromamba-bin/micromamba - micromamba-root-path: ~/micromamba - init-shell: bash + python-version: ${{ matrix.python-version }} + cache: pip - - name: Initialize micromamba from cache (cache hit) - if: steps.cache-restore.outputs.cache-hit == 'true' - run: | - # Make micromamba available to subsequent steps without shell hooks - echo "$HOME/micromamba-bin" >> "$GITHUB_PATH" - echo "MAMBA_ROOT_PREFIX=$HOME/micromamba" >> "$GITHUB_ENV" + - name: Install CPU-only PyTorch + # CPU wheels keep CI fast and avoid pulling multi-GB CUDA builds. + run: pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu - - name: Install pip requirements and CPU-only PyTorch - if: steps.cache-restore.outputs.cache-hit != 'true' - run: | - # Pip packages are installed separately so read-only checkouts and CI caches stay reliable. - ~/micromamba-bin/micromamba run -n fridata_env python -m pip install --upgrade pip - ~/micromamba-bin/micromamba run -n fridata_env python -m pip install -r requirements-fridata.txt - # Avoid a second micromamba/libmamba solve mixing pytorch + conda-forge on an - # already-large env (can abort with libsolv solver_addrule assertion, exit 134). - # CPU wheels from PyTorch are the usual CI approach. Install torchvision from the - # same index as torch so C++/dispatch ops (e.g. torchvision::nms) match conda-forge - # transformers alone pulls torchvision that can mismatch a pip-only torch build. - ~/micromamba-bin/micromamba run -n fridata_env python -m pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu - ~/micromamba-bin/micromamba run -n fridata_env python -m pip install esm - - - name: Save micromamba environment cache - if: steps.cache-restore.outputs.cache-hit != 'true' - uses: actions/cache/save@v4 - with: - path: | - ~/micromamba/envs - ~/micromamba-bin - key: ${{ steps.cache-key.outputs.key }} + - name: Install FRIdata with embeddings + test extras + run: pip install -e ".[embeddings,test]" - name: Run tests - run: | - ~/micromamba-bin/micromamba run -n fridata_env pytest ./tests + run: pytest ./tests From 3ded63f41169663db2b71233b114818756ebce55 Mon Sep 17 00:00:00 2001 From: Adam Nowak Date: Fri, 10 Jul 2026 15:04:04 +0200 Subject: [PATCH 5/8] Tighten MANIFEST.in and drop stale build artifacts - MANIFEST.in now ships only LICENSE/README/example config/scripts and prunes tests, docs, reports and large data files (*.h5/*.cif/*.idx) from the sdist (was 'global-include *', which bundled the whole repo) - Remove committed-on-disk FRIdata.egg-info (stale requires.txt) and __pycache__ - Ignore .DS_Store --- .gitignore | 3 ++- MANIFEST.in | 24 ++++++++++++++++-------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 3bb0260..767ffaa 100644 --- a/.gitignore +++ b/.gitignore @@ -36,4 +36,5 @@ build reports/ -.cursor/**/* \ No newline at end of file +.cursor/**/* +.DS_Store diff --git a/MANIFEST.in b/MANIFEST.in index 9311c11..37bc336 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,9 +1,17 @@ -# include everything in the sdist (be careful — this will bundle all files) -global-include * +# Files to ship in the source distribution (sdist), beyond the packaged Python +# modules. Keep this tight so releases stay small — never bundle test data. +include LICENSE +include README.md +include example.config.json +recursive-include scripts *.sh -recursive-exclude tests * -recursive-exclude .github * -recursive-exclude .pytest_cache * -recursive-exclude build * -recursive-exclude *.egg-info * -recursive-exclude dist * \ No newline at end of file +# Exclude everything that should not ship in a release. +prune tests +prune docs +prune reports +prune .github +prune .gitnexus +prune .cursor +recursive-exclude * __pycache__ +recursive-exclude * *.py[cod] +global-exclude *.h5 *.cif *.idx *.log .DS_Store From 8488e44c018e0761a4ecb2d23b99cab3bfebc7e8 Mon Sep 17 00:00:00 2001 From: Adam Nowak Date: Fri, 10 Jul 2026 15:05:55 +0200 Subject: [PATCH 6/8] Update README and demo notebook for pip/venv workflow - README install section rewritten around python -m venv + pip and the [embeddings] extra; PyPI install instructions added; conda/mamba/libmamba troubleshooting removed - HPC prerequisites updated (Python module + VENV_PATH instead of miniconda) - demo.ipynb cells activate .venv instead of a conda environment --- README.md | 122 +++++++++++++++++++++++++++++++---------------------- demo.ipynb | 70 +++--------------------------- 2 files changed, 78 insertions(+), 114 deletions(-) diff --git a/README.md b/README.md index 904a836..2476231 100644 --- a/README.md +++ b/README.md @@ -7,73 +7,89 @@ [![Source](https://img.shields.io/badge/source-GitHub-303030.svg?style=flat-square)](https://github.com/Tomasz-Lab/FRIdata/) [![GitHub issues](https://img.shields.io/github/issues/Tomasz-Lab/FRIdata.svg?style=flat-square)](https://github.com/Tomasz-Lab/FRIdata/issues) -## Installation and activation +Generate sequences, coordinates, distograms, and embeddings from protein structures at scale. Supports PDB, AFDB, ESMatlas, and local/custom inputs. Full API reference: [docs/index.html](docs/index.html). -1. Download the repo +## How FRIdata works -``` -git clone https://github.com/Tomasz-Lab/FRIdata.git -cd FRIdata -``` +Every dataset is defined by a **database type** (`-d`, `--db`) and a **collection type** (`-c`, `--collection`), plus optional `--proteome` and `--version`. Together they determine the dataset name (for example `AFDB-subset--test`) and how structures are resolved. -2. [Install miniconda](https://www.anaconda.com/docs/getting-started/miniconda/install) +Use `-t` / `--type` to choose what to generate in a run: `sequences`, `coordinates`, `distograms`, `embeddings`, or `all`. -3. Install mamba +### Database type (`-d`, `--db`) -``` -## prioritize 'conda-forge' channel -conda config --add channels conda-forge +Where structures come from. -## update existing packages to use 'conda-forge' channel -conda update -n base --all +| Value | Meaning | +|-------|---------| +| `PDB` | RCSB PDB structures (download by ID) | +| `AFDB` | AlphaFold Database | +| `ESMatlas` | ESM Atlas | +| `other` | Local or custom files via `--input-path` or archives | -## install 'mamba' -conda install -n base mamba -``` +### Collection type (`-c`, `--collection`) -4. Create the environment (recommended) +How much of that source to include. -Use the setup script to create the conda environment, install pip dependencies, and install a PyTorch build matched to your GPU driver: +| Value | Meaning | +|-------|---------| +| `all` | Full database collection | +| `part` | AFDB proteome partition (requires `--proteome`; foldcomp-based) | +| `clust` | AFDB cluster partition (requires `--proteome`; foldcomp-based) | +| `subset` | User-defined ID list via `-i` / `--ids` (optionally `--input-path` for local structures) | -``` -./scripts/setup_env.sh -``` +## Installation -Use a custom environment name for a separate install: +FRIdata is a pure-pip project (Python >= 3.10). No conda/mamba required. -``` -./scripts/setup_env.sh -n fridata_gpu_verify -``` +The core install covers `sequences`, `coordinates` and `distograms`. Embedding +generation needs the heavier `torch`/`esm`/`transformers` stack, which lives in +an optional `embeddings` extra so the default install stays small. + +### Quick start (recommended) -For CPU-only systems: +The setup script creates a virtualenv, installs FRIdata with its dev extras, and +installs a PyTorch build matched to your GPU driver: ``` -./scripts/setup_env.sh --cpu +git clone https://github.com/Tomasz-Lab/FRIdata.git +cd FRIdata +./scripts/setup_env.sh # GPU (auto-detected); use --cpu for CPU-only +source .venv/bin/activate ``` -5. Activate the environment +Options: `--cpu` (CPU-only PyTorch), `--skip-pytorch` (core install, no +embeddings), `-p/--path DIR` (virtualenv location, default `.venv`). + +### Manual installation ``` -# Choose your shell type. Could be one of these: {bash,cmd.exe,dash,fish,nu,posix,powershell,tcsh,xonsh,zsh} -eval "$(mamba shell hook --shell )" -mamba activate fridata_env -``` +python -m venv .venv +source .venv/bin/activate +python -m pip install --upgrade pip -### Manual installation +# Core only (sequences / coordinates / distograms): +pip install -e . + +# ...or with embedding support (torch/esm/transformers from PyPI): +pip install -e ".[embeddings]" +``` -If you prefer to run the steps yourself: +On Linux the default PyPI `torch` wheel is CUDA-enabled. For a specific CUDA +version or CPU-only wheels, use the helper after installing: ``` -mamba env create -f fridata_env_conda.yml -n fridata_env -mamba run -n fridata_env python -m pip install -r requirements-fridata.txt -./scripts/install_pytorch.sh +./scripts/install_pytorch.sh # auto-detect CUDA from the driver +./scripts/install_pytorch.sh --cpu # force CPU-only build ``` -Pip dependencies live in `requirements-fridata.txt` instead of `fridata_env_conda.yml` so environment creation works from read-only repository checkouts. +### Install from PyPI -### Troubleshooting +Once released, FRIdata can be installed without cloning the repo: -If you see `Error opening for writing ".../mambaf..."` while creating the environment, libmamba is trying to write a temporary pip requirements file next to the environment YAML. Use `./scripts/setup_env.sh` instead, or ensure the directory containing `fridata_env_conda.yml` is writable. +``` +pip install fridata # core +pip install "fridata[embeddings]" # with embedding support +``` ## Running tests @@ -81,9 +97,11 @@ If you see `Error opening for writing ".../mambaf..."` while creating the enviro pytest ./tests ``` -## Running on AFDB structures locally +## Usage examples + +### AFDB subset (local) -Requires having a directory with AFDB structures and a text file containing list of AFDB IDs with `\n` delimeter. Assuming all steps from `Installation and activation` succeded +Requires having a directory with AFDB structures and a text file containing list of AFDB IDs with `\n` delimeter. Assuming all steps from [Installation](#installation) succeeded ``` FRIDATA_PATH="" @@ -110,9 +128,9 @@ PYTHONPATH='.' python3 -u ${FRIDATA_PATH}/fridata.py \ For subset runs with `--input-path`, new datasets store canonical keys as `{line_from_ids_file}_{chain}` (for example `A0A2K6V5L6_A`), not the full AlphaFold CIF filename stem. The dataset’s `input_structures.idx` maps each canonical key to the source structure filename. Older datasets created before this convention may still use long AF-style keys. -## Running as a CLI tool +### CLI installation -Assuming all `Installation and activation` steps succeeded. +Assuming all [Installation](#installation) steps succeeded. 0. Go into `FRIdata` directory @@ -134,7 +152,7 @@ fridata <...> (Use ids_file tokens (e.g. plain UniProt) plus chain as the canonical dataset index keys) -## Running on HPC +### HPC (PLGrid / SLURM) Running FRIdata on HPC differs on CPU and GPU nodes. This instruction set is valid for HPC hosted in PLGrid infrastructure. Running on other infrastructures may require additional adjustments. @@ -150,8 +168,8 @@ Prerequisites: - `LAUNCH_WORKER_SLURM_PATH`: path to launch_worker_slurm.sh, defaults to `$DEEPFRI_PATH/FRIdata/scripts/hpc/launch_workers_slurm.sh` - `MEMORY_LIMIT`: memory limit per Dask worker, defaults to `288GiB` - `IP_INTERFACE`: network unix interface, where dask workers are connected. Defaults to `ens1f0` - - `CONDA_ENV_PATH`: path to conda environment, defaults to `$DEEPFRI_PATH/conda_dev` -- Have installed module miniconda3 + - `VENV_PATH`: path to the FRIdata virtualenv, defaults to `$DEEPFRI_PATH/.venv` +- Have a Python module available (`module avail python` — the scripts try `python`/`python3`; adjust the candidate list in `scripts/hpc/*.sh` if your cluster names it differently) - Have installed module gcc Steps: @@ -169,10 +187,10 @@ cd FRIdata chmod u+x -R scripts/hpc/cpu ``` -3. Run `initialize_slurm.sh`. As an argument put the path into directory, where `.conda` directory should be installed and specify `--cpu` flag if the script is run on CPU cluster. +3. Run `initialize_slurm.sh` to create the virtualenv (at `VENV_PATH`, default `$DEEPFRI_PATH/.venv`) and install dependencies. Add the `--cpu` flag on CPU clusters. ``` -./scripts/hpc/initialize_slurm.sh [--cpu] +./scripts/hpc/initialize_slurm.sh [--cpu] ``` 4. Schedule sbatch script into the HPC with all the args specified. Operations to be chosen are: `sequences`, `coordinates`, `embeddings` @@ -186,4 +204,8 @@ sbatch --cpus-per-task= --time= --nodes= --account= --account= --nodes=1 --partition= --cpus-per-task= scripts/hpc/run_slurm.sh embeddings -``` \ No newline at end of file +``` + +## API reference + +See [docs/index.html](docs/index.html) for the full API reference (sub-commands, flags, accepted values) and verbatim `fridata --help` output. \ No newline at end of file diff --git a/demo.ipynb b/demo.ipynb index 4f64507..c7f5696 100644 --- a/demo.ipynb +++ b/demo.ipynb @@ -28,11 +28,7 @@ "cell_type": "markdown", "id": "91f9b031-94bf-4d42-83ce-d433c9677258", "metadata": {}, - "source": [ - "- conda environment is set (accordingly to README.md file)\n", - "- FRIdata CLI tool is created\n", - "- File with IDs" - ] + "source": "- virtualenv is set up and activated (see README.md)\n- FRIdata CLI tool is created\n- File with IDs" }, { "cell_type": "markdown", @@ -48,27 +44,7 @@ "id": "9db9ff88-a90e-4e9a-a2c2-0ad243ad96f6", "metadata": {}, "outputs": [], - "source": [ - "%%bash\n", - "\n", - "export PATH_TO_DATA=\"\"\n", - "export IDS_FILE_PATH=\"\"\n", - "export DASHBOARD_NAME=\"testset_inference_over\"\n", - "export AFDB_PATH=\"$PATH_TO_DATA/afdb/structures\"\n", - "\n", - "eval \"$(conda shell.bash hook)\"\n", - "conda activate fridata_env\n", - "\n", - "fridata generate_data \\\n", - " -t dataset \\\n", - " -d AFDB \\\n", - " -c subset \\\n", - " --overwrite \\\n", - " --version ${DASHBOARD_NAME} \\\n", - " -i ${IDS_FILE_PATH} \\\n", - " --input-path ${AFDB_PATH} \\\n", - " -e \"esm2_t33_650M_UR50D\"" - ] + "source": "%%bash\n\nexport PATH_TO_DATA=\"\"\nexport IDS_FILE_PATH=\"\"\nexport DASHBOARD_NAME=\"testset_inference_over\"\nexport AFDB_PATH=\"$PATH_TO_DATA/afdb/structures\"\n\nsource .venv/bin/activate\n\nfridata generate_data \\\n -t dataset \\\n -d AFDB \\\n -c subset \\\n --overwrite \\\n --version ${DASHBOARD_NAME} \\\n -i ${IDS_FILE_PATH} \\\n --input-path ${AFDB_PATH} \\\n -e \"esm2_t33_650M_UR50D\"" }, { "cell_type": "markdown", @@ -84,19 +60,7 @@ "id": "4203f2e3-3f3d-4a80-8cb3-4ed3b6b9c9ba", "metadata": {}, "outputs": [], - "source": [ - "%%bash\n", - "\n", - "# Path to dataset.json generated from 1. task\n", - "export PATH_TO_DATA=\"\"\n", - "export PATH_TO_DATASET_JSON=\"$PATH_TO_DATA/datasets/AFDB-subset--testset_inference_over/dataset.json\"\n", - "\n", - "eval \"$(conda shell.bash hook)\"\n", - "conda activate fridata_env\n", - "\n", - "fridata create_dashboard \\\n", - " --dataset ${PATH_TO_DATASET_JSON}" - ] + "source": "%%bash\n\n# Path to dataset.json generated from 1. task\nexport PATH_TO_DATA=\"\"\nexport PATH_TO_DATASET_JSON=\"$PATH_TO_DATA/datasets/AFDB-subset--testset_inference_over/dataset.json\"\n\nsource .venv/bin/activate\n\nfridata create_dashboard \\\n --dataset ${PATH_TO_DATASET_JSON}" }, { "cell_type": "markdown", @@ -155,16 +119,7 @@ "id": "ea69cf70-2c29-47d4-a6d8-5153c7a9b1b3", "metadata": {}, "outputs": [], - "source": [ - "%%bash\n", - "\n", - "export PATH_TO_DATA=\"\"\n", - "\n", - "eval \"$(conda shell.bash hook)\"\n", - "conda activate fridata_env\n", - "\n", - "fridata load --file-path \"$PATH_TO_DATA/datasets/AFDB-subset--testset_inference_over\"" - ] + "source": "%%bash\n\nexport PATH_TO_DATA=\"\"\n\nsource .venv/bin/activate\n\nfridata load --file-path \"$PATH_TO_DATA/datasets/AFDB-subset--testset_inference_over\"" }, { "cell_type": "markdown", @@ -198,20 +153,7 @@ "id": "a5fe4827-1f06-4a27-874b-b26b158ad784", "metadata": {}, "outputs": [], - "source": [ - "%%bash\n", - "\n", - "export PATH_TO_DATA=\"\"\n", - "export PATH_TO_STRUCTURES=\"$PATH_TO_DATA/structures/AFDB/subset_/testset_inference_over/0/pdbs.h5\"\n", - "export PATH_TO_EMBEDDINGS=\"$PATH_TO_DATA/embeddings/AFDB-subset--_test_dask/batch_0.h5\"\n", - "export PATH_TO_DISTOGRAMS=\"$PATH_TO_DATA/distograms/AFDB-subset--_test_dask/batch_0.h5\"\n", - "export PATH_TO_COORDINATES=\"$PATH_TO_DATA/coordinates/AFDB-subset--_test_dask/batch_0_ca.h5\"\n", - "\n", - "eval \"$(conda shell.bash hook)\"\n", - "conda activate fridata_env\n", - "\n", - "fridata inspect_h5 --help" - ] + "source": "%%bash\n\nexport PATH_TO_DATA=\"\"\nexport PATH_TO_STRUCTURES=\"$PATH_TO_DATA/structures/AFDB/subset_/testset_inference_over/0/pdbs.h5\"\nexport PATH_TO_EMBEDDINGS=\"$PATH_TO_DATA/embeddings/AFDB-subset--_test_dask/batch_0.h5\"\nexport PATH_TO_DISTOGRAMS=\"$PATH_TO_DATA/distograms/AFDB-subset--_test_dask/batch_0.h5\"\nexport PATH_TO_COORDINATES=\"$PATH_TO_DATA/coordinates/AFDB-subset--_test_dask/batch_0_ca.h5\"\n\nsource .venv/bin/activate\n\nfridata inspect_h5 --help" } ], "metadata": { @@ -235,4 +177,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} +} \ No newline at end of file From c804e53256f10f9c00a79f3a2999ce01e1080802 Mon Sep 17 00:00:00 2001 From: Adam Nowak Date: Fri, 10 Jul 2026 15:06:17 +0200 Subject: [PATCH 7/8] Remove conda env files; add PyPI Trusted Publishing workflow - Delete fridata_env_conda.yml and requirements-fridata.txt (deps now live solely in pyproject.toml) - Add .github/workflows/publish.yml: build sdist+wheel, twine check, and publish to PyPI via OIDC Trusted Publishing on GitHub Release --- .github/workflows/publish.yml | 49 +++++++++++++++++++++++++++++++++++ fridata_env_conda.yml | 18 ------------- requirements-fridata.txt | 12 --------- 3 files changed, 49 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/publish.yml delete mode 100644 fridata_env_conda.yml delete mode 100644 requirements-fridata.txt diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..333e96f --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,49 @@ +name: Publish to PyPI + +# Build the sdist + wheel and publish to PyPI when a GitHub Release is published. +# Uses PyPI Trusted Publishing (OIDC) — no API tokens/secrets to manage. +# One-time setup: on PyPI, add a "trusted publisher" for this repo pointing at +# this workflow file and the `pypi` environment. +on: + release: + types: [published] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Build sdist and wheel + run: | + python -m pip install --upgrade build + python -m build + + - name: Check distributions + run: | + python -m pip install --upgrade twine + twine check dist/* + + - uses: actions/upload-artifact@v4 + with: + name: dist + path: dist/ + + publish: + needs: build + runs-on: ubuntu-latest + environment: pypi + permissions: + id-token: write # required for Trusted Publishing (OIDC) + steps: + - uses: actions/download-artifact@v4 + with: + name: dist + path: dist/ + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/fridata_env_conda.yml b/fridata_env_conda.yml deleted file mode 100644 index 1ffa364..0000000 --- a/fridata_env_conda.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: fridata_env -channels: - - conda-forge - - defaults -dependencies: - - python==3.10 - - pip - - dask>=2024.4.2,<2025.0.0 - - dask[dataframe] - - distributed>=2024.4.2,<2025.0.0 - - python-dotenv - - tables - - biopython - - bokeh>=2.4.2,<3.0.0 - - pdbfixer - - mamba - - transformers - - tqdm diff --git a/requirements-fridata.txt b/requirements-fridata.txt deleted file mode 100644 index 89ef7f1..0000000 --- a/requirements-fridata.txt +++ /dev/null @@ -1,12 +0,0 @@ -foldcomp -pydantic>=2.8.2 -scipy>=1.14.0 -h5py -matplotlib -biotite>=0.41.2 -numpy>=1.26.4,<1.27.0 -genson -bs4 -bio~=1.7.0 -pytest -pytest-order From 501d67fa7f0c439f1c0bf78159d4cc6ac06c6450 Mon Sep 17 00:00:00 2001 From: Adam Nowak Date: Fri, 10 Jul 2026 15:10:00 +0200 Subject: [PATCH 8/8] Skip embedding tests gracefully when torch is not installed Core-only checkouts (pip install -e .[test] without [embeddings]) previously failed the entire pytest run at collection time on a top-level 'import torch'. Guard with pytest.importorskip so the rest of the suite still runs. --- tests/test_embeddings.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_embeddings.py b/tests/test_embeddings.py index 8a42ed2..0f2da04 100644 --- a/tests/test_embeddings.py +++ b/tests/test_embeddings.py @@ -9,6 +9,11 @@ from os.path import join +# Embedding tests need the optional 'embeddings' extra (torch/esm/transformers). +# Skip the whole module gracefully when it is not installed so a core-only +# `pip install -e .[test]` checkout can still run the rest of the suite. +pytest.importorskip("torch", reason="install FRIdata[embeddings] to run embedding tests") + from toolbox.models.embedding.embedder.esm2_embedder import ESM2Embedder from tests.utils import compare_pdb_files from pathlib import Path