Migrate conda to pip

.github/workflows/ci.yml

@@ -6,74 +6,27 @@ on:
 jobs:
   test:
     runs-on: ubuntu-latest
-    defaults:
-      run:
-        shell: bash -el {0}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
 
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Generate cache key
-        id: cache-key
-        run: |
-          # Create a hash from the environment files and PyTorch CPU install string
-          ENV_HASH=$(cat fridata_env_conda.yml requirements-fridata.txt | sha256sum | cut -d' ' -f1)
-          PYTORCH_SPEC="pip-torch-torchvision-cpu-whl"
-          CACHE_KEY="micromamba-${{ runner.os }}-${ENV_HASH}-${PYTORCH_SPEC}"
-          echo "key=${CACHE_KEY}" >> $GITHUB_OUTPUT
-
-      - name: Restore micromamba environment cache
-        id: cache-restore
-        uses: actions/cache/restore@v4
-        with:
-          path: |
-            ~/micromamba/envs
-            ~/micromamba-bin
-          key: ${{ steps.cache-key.outputs.key }}
-
-      - name: Setup micromamba (cache miss)
-        if: steps.cache-restore.outputs.cache-hit != 'true'
-        uses: mamba-org/setup-micromamba@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
         with:
-          environment-file: fridata_env_conda.yml
-          environment-name: fridata_env
-          cache-environment: false
-          cache-downloads: false
-          micromamba-binary-path: ~/micromamba-bin/micromamba
-          micromamba-root-path: ~/micromamba
-          init-shell: bash
+          python-version: ${{ matrix.python-version }}
+          cache: pip
 
-      - name: Initialize micromamba from cache (cache hit)
-        if: steps.cache-restore.outputs.cache-hit == 'true'
-        run: |
-          # Make micromamba available to subsequent steps without shell hooks
-          echo "$HOME/micromamba-bin" >> "$GITHUB_PATH"
-          echo "MAMBA_ROOT_PREFIX=$HOME/micromamba" >> "$GITHUB_ENV"
+      - name: Install CPU-only PyTorch
+        # CPU wheels keep CI fast and avoid pulling multi-GB CUDA builds.
+        run: pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu
 
-      - name: Install pip requirements and CPU-only PyTorch
-        if: steps.cache-restore.outputs.cache-hit != 'true'
-        run: |
-          # Pip packages are installed separately so read-only checkouts and CI caches stay reliable.
-          ~/micromamba-bin/micromamba run -n fridata_env python -m pip install --upgrade pip
-          ~/micromamba-bin/micromamba run -n fridata_env python -m pip install -r requirements-fridata.txt
-          # Avoid a second micromamba/libmamba solve mixing pytorch + conda-forge on an
-          # already-large env (can abort with libsolv solver_addrule assertion, exit 134).
-          # CPU wheels from PyTorch are the usual CI approach. Install torchvision from the
-          # same index as torch so C++/dispatch ops (e.g. torchvision::nms) match conda-forge
-          # transformers alone pulls torchvision that can mismatch a pip-only torch build.
-          ~/micromamba-bin/micromamba run -n fridata_env python -m pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu
-          ~/micromamba-bin/micromamba run -n fridata_env python -m pip install esm
-
-      - name: Save micromamba environment cache
-        if: steps.cache-restore.outputs.cache-hit != 'true'
-        uses: actions/cache/save@v4
-        with:
-          path: |
-            ~/micromamba/envs
-            ~/micromamba-bin
-          key: ${{ steps.cache-key.outputs.key }}
+      - name: Install FRIdata with embeddings + test extras
+        run: pip install -e ".[embeddings,test]"
 
       - name: Run tests
-        run: |
-          ~/micromamba-bin/micromamba run -n fridata_env pytest ./tests
+        run: pytest ./tests

.github/workflows/publish.yml

@@ -0,0 +1,49 @@
+name: Publish to PyPI
+
+# Build the sdist + wheel and publish to PyPI when a GitHub Release is published.
+# Uses PyPI Trusted Publishing (OIDC) — no API tokens/secrets to manage.
+# One-time setup: on PyPI, add a "trusted publisher" for this repo pointing at
+# this workflow file and the `pypi` environment.
+on:
+  release:
+    types: [published]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Build sdist and wheel
+        run: |
+          python -m pip install --upgrade build
+          python -m build
+
+      - name: Check distributions
+        run: |
+          python -m pip install --upgrade twine
+          twine check dist/*
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write  # required for Trusted Publishing (OIDC)
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

.gitignore

@@ -36,4 +36,5 @@ build
 
 reports/
 
-.cursor/**/*
+.cursor/**/*
+.DS_Store

MANIFEST.in

@@ -1,9 +1,17 @@
-# include everything in the sdist (be careful — this will bundle all files)
-global-include *
+# Files to ship in the source distribution (sdist), beyond the packaged Python
+# modules. Keep this tight so releases stay small — never bundle test data.
+include LICENSE
+include README.md
+include example.config.json
+recursive-include scripts *.sh
 
-recursive-exclude tests *
-recursive-exclude .github *
-recursive-exclude .pytest_cache *
-recursive-exclude build *
-recursive-exclude *.egg-info *
-recursive-exclude dist *
+# Exclude everything that should not ship in a release.
+prune tests
+prune docs
+prune reports
+prune .github
+prune .gitnexus
+prune .cursor
+recursive-exclude * __pycache__
+recursive-exclude * *.py[cod]
+global-exclude *.h5 *.cif *.idx *.log .DS_Store

README.md

@@ -7,83 +7,101 @@
 [![Source](https://img.shields.io/badge/source-GitHub-303030.svg?style=flat-square)](https://github.com/Tomasz-Lab/FRIdata/)
 [![GitHub issues](https://img.shields.io/github/issues/Tomasz-Lab/FRIdata.svg?style=flat-square)](https://github.com/Tomasz-Lab/FRIdata/issues)
 
-## Installation and activation
+Generate sequences, coordinates, distograms, and embeddings from protein structures at scale. Supports PDB, AFDB, ESMatlas, and local/custom inputs. Full API reference: [docs/index.html](docs/index.html).
 
-1. Download the repo
+## How FRIdata works
 
-```
-git clone https://github.com/Tomasz-Lab/FRIdata.git
-cd FRIdata
-```
+Every dataset is defined by a **database type** (`-d`, `--db`) and a **collection type** (`-c`, `--collection`), plus optional `--proteome` and `--version`. Together they determine the dataset name (for example `AFDB-subset--test`) and how structures are resolved.
 
-2. [Install miniconda](https://www.anaconda.com/docs/getting-started/miniconda/install)
+Use `-t` / `--type` to choose what to generate in a run: `sequences`, `coordinates`, `distograms`, `embeddings`, or `all`.
 
-3. Install mamba
+### Database type (`-d`, `--db`)
 
-```
-## prioritize 'conda-forge' channel
-conda config --add channels conda-forge
+Where structures come from.
 
-## update existing packages to use 'conda-forge' channel
-conda update -n base --all
+| Value | Meaning |
+|-------|---------|
+| `PDB` | RCSB PDB structures (download by ID) |
+| `AFDB` | AlphaFold Database |
+| `ESMatlas` | ESM Atlas |
+| `other` | Local or custom files via `--input-path` or archives |
 
-## install 'mamba'
-conda install -n base mamba
-```
+### Collection type (`-c`, `--collection`)
 
-4. Create the environment (recommended)
+How much of that source to include.
 
-Use the setup script to create the conda environment, install pip dependencies, and install a PyTorch build matched to your GPU driver:
+| Value | Meaning |
+|-------|---------|
+| `all` | Full database collection |
+| `part` | AFDB proteome partition (requires `--proteome`; foldcomp-based) |
+| `clust` | AFDB cluster partition (requires `--proteome`; foldcomp-based) |
+| `subset` | User-defined ID list via `-i` / `--ids` (optionally `--input-path` for local structures) |
 
-```
-./scripts/setup_env.sh
-```
+## Installation
 
-Use a custom environment name for a separate install:
+FRIdata is a pure-pip project (Python >= 3.10). No conda/mamba required.
 
-```
-./scripts/setup_env.sh -n fridata_gpu_verify
-```
+The core install covers `sequences`, `coordinates` and `distograms`. Embedding
+generation needs the heavier `torch`/`esm`/`transformers` stack, which lives in
+an optional `embeddings` extra so the default install stays small.
+
+### Quick start (recommended)
 
-For CPU-only systems:
+The setup script creates a virtualenv, installs FRIdata with its dev extras, and
+installs a PyTorch build matched to your GPU driver:
 
 ```
-./scripts/setup_env.sh --cpu
+git clone https://github.com/Tomasz-Lab/FRIdata.git
+cd FRIdata
+./scripts/setup_env.sh            # GPU (auto-detected); use --cpu for CPU-only
+source .venv/bin/activate
 ```
 
-5. Activate the environment
+Options: `--cpu` (CPU-only PyTorch), `--skip-pytorch` (core install, no
+embeddings), `-p/--path DIR` (virtualenv location, default `.venv`).
+
+### Manual installation
 
 ```
-# Choose your shell type. Could be one of these: {bash,cmd.exe,dash,fish,nu,posix,powershell,tcsh,xonsh,zsh}
-eval "$(mamba shell hook --shell <replace with shell type>)"
-mamba activate fridata_env
-```
+python -m venv .venv
+source .venv/bin/activate
+python -m pip install --upgrade pip
 
-### Manual installation
+# Core only (sequences / coordinates / distograms):
+pip install -e .
+
+# ...or with embedding support (torch/esm/transformers from PyPI):
+pip install -e ".[embeddings]"
+```
 
-If you prefer to run the steps yourself:
+On Linux the default PyPI `torch` wheel is CUDA-enabled. For a specific CUDA
+version or CPU-only wheels, use the helper after installing:
 
 ```
-mamba env create -f fridata_env_conda.yml -n fridata_env
-mamba run -n fridata_env python -m pip install -r requirements-fridata.txt
-./scripts/install_pytorch.sh
+./scripts/install_pytorch.sh          # auto-detect CUDA from the driver
+./scripts/install_pytorch.sh --cpu    # force CPU-only build
 ```
 
-Pip dependencies live in `requirements-fridata.txt` instead of `fridata_env_conda.yml` so environment creation works from read-only repository checkouts.
+### Install from PyPI
 
-### Troubleshooting
+Once released, FRIdata can be installed without cloning the repo:
 
-If you see `Error opening for writing ".../mambaf..."` while creating the environment, libmamba is trying to write a temporary pip requirements file next to the environment YAML. Use `./scripts/setup_env.sh` instead, or ensure the directory containing `fridata_env_conda.yml` is writable.
+```
+pip install fridata                 # core
+pip install "fridata[embeddings]"   # with embedding support
+```
 
 ## Running tests
 
 ```
 pytest ./tests
 ```
 
-## Running on AFDB structures locally
+## Usage examples
+
+### AFDB subset (local)
 
-Requires having a directory with AFDB structures and a text file containing list of AFDB IDs with `\n` delimeter. Assuming all steps from `Installation and activation` succeded
+Requires having a directory with AFDB structures and a text file containing list of AFDB IDs with `\n` delimeter. Assuming all steps from [Installation](#installation) succeeded
 
 ```
 FRIDATA_PATH="<repository path>"
@@ -110,9 +128,9 @@ PYTHONPATH='.' python3 -u ${FRIDATA_PATH}/fridata.py \
 
 For subset runs with `--input-path`, new datasets store canonical keys as `{line_from_ids_file}_{chain}` (for example `A0A2K6V5L6_A`), not the full AlphaFold CIF filename stem. The dataset’s `input_structures.idx` maps each canonical key to the source structure filename. Older datasets created before this convention may still use long AF-style keys.
 
-## Running as a CLI tool
+### CLI installation
 
-Assuming all `Installation and activation` steps succeeded.
+Assuming all [Installation](#installation) steps succeeded.
 
 0. Go into `FRIdata` directory
 
@@ -134,7 +152,7 @@ fridata <...>
 
 (Use ids_file tokens (e.g. plain UniProt) plus chain as the canonical dataset index keys)
 
-## Running on HPC
+### HPC (PLGrid / SLURM)
 
 Running FRIdata on HPC differs on CPU and GPU nodes. This instruction set is valid for HPC hosted in PLGrid infrastructure. Running on other infrastructures may require additional adjustments.
 
@@ -150,8 +168,8 @@ Prerequisites:
         - `LAUNCH_WORKER_SLURM_PATH`: path to launch_worker_slurm.sh, defaults to `$DEEPFRI_PATH/FRIdata/scripts/hpc/launch_workers_slurm.sh`
         - `MEMORY_LIMIT`: memory limit per Dask worker, defaults to `288GiB`
         - `IP_INTERFACE`: network unix interface, where dask workers are connected. Defaults to `ens1f0`
-        - `CONDA_ENV_PATH`: path to conda environment, defaults to `$DEEPFRI_PATH/conda_dev`
-- Have installed module miniconda3
+        - `VENV_PATH`: path to the FRIdata virtualenv, defaults to `$DEEPFRI_PATH/.venv`
+- Have a Python module available (`module avail python` — the scripts try `python`/`python3`; adjust the candidate list in `scripts/hpc/*.sh` if your cluster names it differently)
 - Have installed module gcc
 
 Steps:
@@ -169,10 +187,10 @@ cd FRIdata
 chmod u+x -R scripts/hpc/cpu
 ```
 
-3. Run `initialize_slurm.sh`. As an argument put the path into directory, where `.conda` directory should be installed and specify `--cpu` flag if the script is run on CPU cluster.
+3. Run `initialize_slurm.sh` to create the virtualenv (at `VENV_PATH`, default `$DEEPFRI_PATH/.venv`) and install dependencies. Add the `--cpu` flag on CPU clusters.
 
 ```
-./scripts/hpc/initialize_slurm.sh <path to .conda> [--cpu]
+./scripts/hpc/initialize_slurm.sh [--cpu]
 ```
 
 4. Schedule sbatch script into the HPC with all the args specified. Operations to be chosen are: `sequences`, `coordinates`, `embeddings`
@@ -186,4 +204,8 @@ sbatch --cpus-per-task=<cpus> --time=<HH:MM:SS> --nodes=<nodes> --account=<grant
 For GPU:
 ```
 sbatch --gres=gpu[:gpu-number] --time=<HH:MM:SS> --account=<grant name> --nodes=1 --partition=<partition name> --cpus-per-task=<cpus> scripts/hpc/run_slurm.sh embeddings
-```
+```
+
+## API reference
+
+See [docs/index.html](docs/index.html) for the full API reference (sub-commands, flags, accepted values) and verbatim `fridata <subcommand> --help` output.