Adding windows support (cytomining#698)

* add windows to integration test

* fixes to enable windows support

* windows compatibility

* skip python 3.14 on windows

* wrap in try finally to avoid leaks

* simplify sql query

* solve csv sniff issue with load_platemap for windows os

* solve windows csv sniffer for load_profiles

* move to parquet writing in cli, add legacy csv test

* add comments and fix tests for external metadata that are csvs

* [pre-commit.ci lite] apply automatic fixes

* data are now parquet by default, modernize

* clean up test load

---------

Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com>

Author: gwaybio

.github/workflows/integration-test.yml

@@ -113,7 +113,15 @@ jobs:
         os:
           - macos-14
           - ubuntu-24.04
+          - windows-2022
         python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
+        exclude:
+          # NumPy's MINGW-W64 Windows build for Python 3.14 is self-described as
+          # "experimental" with crashes expected. Exclude until NumPy ships a
+          # stable Windows wheel for 3.14.
+          # https://github.com/numpy/numpy/issues/26038
+          - os: windows-2022
+            python-version: "3.14"
     runs-on: ${{ matrix.os }}
     env:
       OS: ${{ matrix.os }}

pycytominer/annotate.py

@@ -34,6 +34,7 @@ def annotate(
     compression_options: Optional[Union[str, dict[str, str]]] = None,
     float_format: Optional[str] = None,
     cmap_args: Optional[dict[str, Union[str]]] = None,
+    platemap_sep: Optional[str] = None,
     **kwargs,
 ) -> Union[pd.DataFrame, str]:
     """Add metadata to aggregated profiles.
@@ -64,7 +65,9 @@ def annotate(
     external_metadata : pd.DataFrame or file, optional
         DataFrame or file with additional metadata information.
         Most common use case is a QC.parquet file with QC flags for each profile
-        that comes from coSMicQC.
+        that comes from coSMicQC. File paths are loaded via :func:`load_profiles`;
+        on Windows, CSV/TSV files are not supported — pass a Parquet file or a
+        pre-loaded DataFrame instead (see the Windows note in :func:`load_profiles`).
     external_join_on : str or list, optional
         Merge column(s) shared by the annotated profiles and external metadata.
         When provided, these keys are used on both sides of the external merge.
@@ -77,6 +80,14 @@ def annotate(
         decimal precision.
     cmap_args : dict, default None
         Potential keyword arguments for annotate_cmap(). See cyto_utils/annotate_custom.py for more details.
+    platemap_sep : str, optional
+        Column delimiter for the platemap file (e.g. ``","`` for CSV, ``"\\t"``
+        for TSV). Only applies when ``platemap`` is a file path — ignored when
+        a DataFrame is passed directly.
+
+        When ``None`` (the default), the delimiter is detected automatically.
+        Automatic detection can be unreliable on Windows for tab-separated files;
+        pass ``platemap_sep="\\t"`` explicitly in that case.
 
     Returns
     -------
@@ -91,7 +102,7 @@ def annotate(
 
     # Load Data
     profiles = load_profiles(profiles)
-    platemap = load_platemap(platemap, add_metadata_id_to_platemap)
+    platemap = load_platemap(platemap, add_metadata_id_to_platemap, sep=platemap_sep)
 
     annotated = platemap.merge(
         profiles,

pycytominer/cyto_utils/DeepProfiler_processing.py

@@ -110,14 +110,18 @@ def extract_filename_metadata(
         loc : dict
             dict with metadata
         """
+        npz_path = pathlib.PurePath(npz_file)
         if delimiter == "/":
-            site = str(npz_file).split("/")[-1].strip(".npz")
-            well = str(npz_file).split("/")[-2]
+            # Layout: .../plate/well/site.npz
+            site = npz_path.stem
+            well = npz_path.parent.name
+            plate = npz_path.parent.parent.name
         else:
-            base_file = os.path.basename(npz_file).strip(".npz").split(delimiter)
+            # Layout: .../plate/well{delimiter}site.npz
+            base_file = npz_path.stem.split(delimiter)
             site = base_file[-1]
             well = base_file[-2]
-        plate = str(npz_file).split("/")[-2]
+            plate = npz_path.parent.name
 
         loc = {"site": site, "well": well, "plate": plate}
         return loc

pycytominer/cyto_utils/cell_locations.py

@@ -3,6 +3,7 @@
 """
 
 import collections
+import os
 import pathlib
 import tempfile
 from typing import Optional, Union
@@ -171,19 +172,28 @@ def _download_s3(self, uri: str):
 
         bucket, key = self._parse_s3_path(uri)
 
-        with tempfile.NamedTemporaryFile(
-            delete=False, suffix=pathlib.Path(key).name
-        ) as tmp_file:
-            self.s3.download_file(bucket, key, tmp_file.name)
+        # Use mkstemp so the file descriptor is closed before boto3 touches it.
+        # NamedTemporaryFile holds an exclusive OS lock while open on Windows;
+        # s3transfer does os.remove() + rename onto the same path internally,
+        # which raises PermissionError [WinError 32] if the fd is still open.
+        fd, tmp_path = tempfile.mkstemp(suffix=pathlib.Path(key).name)
+        os.close(fd)
 
-            # Check if the downloaded file exists and has a size greater than 0
-            tmp_file_path = pathlib.Path(tmp_file.name)
+        try:
+            self.s3.download_file(bucket, key, tmp_path)
+
+            tmp_file_path = pathlib.Path(tmp_path)
             if tmp_file_path.exists() and tmp_file_path.stat().st_size > 0:
-                return tmp_file.name
-            else:
-                raise ValueError(
-                    f"Downloaded file '{tmp_file.name}' is empty or does not exist."
-                )
+                return tmp_path
+
+            raise ValueError(
+                f"Downloaded file '{tmp_path}' is empty or does not exist."
+            )
+        except Exception:
+            tmp_file_path = pathlib.Path(tmp_path)
+            if tmp_file_path.exists():
+                tmp_file_path.unlink()
+            raise
 
     def _load_metadata(self):
         """Load the metadata into a Pandas DataFrame
@@ -358,38 +368,39 @@ def _get_joined_image_nuclei_tables(self):
         # get the sqlalchemy.engine.Engine object for the single_cell file
         temp_single_cell_input, engine = self._get_single_cell_engine()
 
-        # check that the single_cell file has the required tables and columns
-        self._check_single_cell_correctness(engine)
-
-        image_index_str = ", ".join(self.image_key)
-
-        # merge the Image and Nuclei tables in SQL
-
-        join_query = f"""
-        SELECT Nuclei.{self.table_column},Nuclei.{self.image_column},Nuclei.{self.object_column},Nuclei.{self.cell_x_loc},Nuclei.{self.cell_y_loc},Image.{image_index_str}
-        FROM Nuclei
-        INNER JOIN Image
-        ON Nuclei.{self.image_column} = Image.{self.image_column} and Nuclei.{self.table_column} = Image.{self.table_column};
-        """
-
-        column_types = {
-            self.image_column: "int64",
-            self.table_column: "int64",
-            self.object_column: "int64",
-            self.cell_x_loc: "float",
-            self.cell_y_loc: "float",
-        }
-
-        for image_key in self.image_key:
-            column_types[image_key] = "str"
-
-        joined_df = pd.read_sql_query(join_query, engine, dtype=column_types)
-
-        # if the single_cell file was downloaded from S3, delete the temporary file
-        if temp_single_cell_input is not None:
-            pathlib.Path(temp_single_cell_input).unlink()
-
-        return joined_df
+        try:
+            # check that the single_cell file has the required tables and columns
+            self._check_single_cell_correctness(engine)
+
+            # CAST each column at the database level.
+            # SQLite uses type affinity rather than strict column types, so
+            # values may not match their declared type; CAST enforces the
+            # expected types in the query itself rather than via a slower
+            # post-hoc pandas dtype conversion.
+            join_query = f"""
+            SELECT
+                CAST(Nuclei.{self.table_column} AS INTEGER) AS {self.table_column},
+                CAST(Nuclei.{self.image_column} AS INTEGER) AS {self.image_column},
+                CAST(Nuclei.{self.object_column} AS INTEGER) AS {self.object_column},
+                CAST(Nuclei.{self.cell_x_loc} AS REAL) AS {self.cell_x_loc},
+                CAST(Nuclei.{self.cell_y_loc} AS REAL) AS {self.cell_y_loc},
+                {", ".join(f"CAST(Image.{k} AS TEXT) AS {k}" for k in self.image_key)}
+            FROM Nuclei
+            INNER JOIN Image
+                ON Nuclei.{self.image_column} = Image.{self.image_column}
+                AND Nuclei.{self.table_column} = Image.{self.table_column};
+            """
+
+            return pd.read_sql_query(join_query, engine)
+        finally:
+            # Always dispose the engine and remove the temp file.
+            # On Windows, SQLAlchemy's connection pool keeps the SQLite file open;
+            # unlink() raises PermissionError [WinError 32] unless disposed first.
+            engine.dispose()
+            if temp_single_cell_input is not None:
+                temp_path = pathlib.Path(temp_single_cell_input)
+                if temp_path.exists():
+                    temp_path.unlink()
 
     def _load_single_cell(self):
         """Load the required columns from the `Image` and `Nuclei` tables in the single_cell file or sqlalchemy.engine.Engine object into a Pandas DataFrame

pycytominer/cyto_utils/collate.py

@@ -240,7 +240,13 @@ def collate(
         add_image_features=add_image_features,
         image_feature_categories=image_feature_categories,
     )
-    database.aggregate_profiles(output_file=str(aggregated_file))
+    try:
+        database.aggregate_profiles(output_file=str(aggregated_file))
+    finally:
+        # Release the SQLite connection so the file can be removed or renamed on
+        # Windows (which holds an exclusive lock on open database files).
+        database.conn.close()
+        database.engine.dispose()
 
     if aws_remote:
         if printtoscreen:

pycytominer/cyto_utils/load.py

@@ -5,6 +5,7 @@
 import csv
 import gzip
 import pathlib
+import sys
 from typing import Any, Optional, Union
 
 import numpy as np
@@ -390,33 +391,63 @@ def load_profiles(
         if anndata_type := is_anndata(profiles):
             return read_anndata(profiles, anndata_type)
 
-    # otherwise, assume its a csv/tsv file and infer the delimiter
+    # CSV/TSV fallback — not supported on Windows.
+    # Python's csv.Sniffer unreliably detects tab-separated files on Windows
+    # (https://github.com/python/cpython/issues/119123), and there is no
+    # safe way to auto-detect the delimiter cross-platform without risking
+    # silently loading data with the wrong separator.
+    if sys.platform == "win32":
+        raise OSError(
+            "Loading CSV/TSV profiles via automatic delimiter detection is not "
+            "supported on Windows (see https://github.com/python/cpython/issues/119123).\n"
+            "We recommend reprocessing your CellProfiler output with CytoTable "
+            "(https://github.com/cytomining/CytoTable) to produce a standardised "
+            "Parquet file, which is supported on all platforms and is the preferred "
+            "input format for pycytominer.\n"
+            "If reprocessing is not an option, load the file manually with "
+            "pd.read_csv(path, sep=',') or pd.read_csv(path, sep='\\t') and pass "
+            "the resulting DataFrame directly."
+        )
     delim = infer_delim(profiles)
     return pd.read_csv(str(profiles), sep=delim)
 
 
 def load_platemap(
-    platemap: Union[str, pd.DataFrame], add_metadata_id=True
+    platemap: Union[str, pd.DataFrame],
+    add_metadata_id: bool = True,
+    sep: Optional[str] = None,
 ) -> pd.DataFrame:
     """
-    Unless a dataframe is provided, load the given platemap dataframe from path or string
+    Unless a dataframe is provided, load the given platemap dataframe from path or string.
 
     Parameters
     ----------
     platemap : pd.DataFrame or str
-        location or actual pd.DataFrame of platemap file
+        Location or actual pd.DataFrame of platemap file.
+
+    add_metadata_id : bool, default True
+        Whether ``Metadata_`` should be prepended to all platemap columns.
 
-    add_metadata_id : bool
-        boolean if ``Metadata_`` should be appended to all platemap columns
+    sep : str, optional
+        The column delimiter used in the platemap file (e.g. ``","`` for CSV,
+        ``"\\t"`` for TSV). Only relevant when ``platemap`` is a file path rather
+        than a DataFrame — has no effect when a DataFrame is passed directly.
+
+        When ``None`` (the default), the delimiter is detected automatically via
+        :func:`infer_delim`. Automatic detection relies on Python's
+        ``csv.Sniffer``, which can be unreliable on Windows for tab-separated
+        files (see `cpython#119123
+        <https://github.com/python/cpython/issues/119123>`_). If you are on
+        Windows and loading a TSV platemap, pass ``sep="\\t"`` explicitly.
 
     Returns
     -------
     platemap : pd.DataFrame
-        pandas DataFrame of profiles
+        pandas DataFrame of platemap.
     """
     if not isinstance(platemap, pd.DataFrame):
         try:
-            delim = infer_delim(platemap)
+            delim = sep if sep is not None else infer_delim(platemap)
             platemap = pd.read_csv(platemap, sep=delim)
         except FileNotFoundError:
             raise FileNotFoundError(f"{platemap} platemap file not found")

tests/test_annotate.py

@@ -61,6 +61,40 @@ def test_annotate():
     pd.testing.assert_frame_equal(result, expected_result)
 
 
+def test_annotate_platemap_sep(tmp_path):
+    """platemap_sep bypasses infer_delim — runs on all platforms including Windows."""
+    # Write platemap as TSV and CSV to tmp_path
+    tsv_path = tmp_path / "platemap.tsv"
+    csv_path = tmp_path / "platemap.csv"
+    PLATEMAP_DF.to_csv(tsv_path, sep="\t", index=False)
+    PLATEMAP_DF.to_csv(csv_path, sep=",", index=False)
+
+    expected_result = (
+        PLATEMAP_DF
+        .merge(DATA_DF, left_on="well_position", right_on="Metadata_Well")
+        .rename(columns={"gene": "Metadata_gene"})
+        .drop("well_position", axis="columns")
+    )
+
+    # TSV platemap loaded via explicit platemap_sep
+    result_tsv = annotate(
+        profiles=DATA_DF,
+        platemap=str(tsv_path),
+        join_on=["Metadata_well_position", "Metadata_Well"],
+        platemap_sep="\t",
+    )
+    pd.testing.assert_frame_equal(result_tsv, expected_result)
+
+    # CSV platemap loaded via explicit platemap_sep
+    result_csv = annotate(
+        profiles=DATA_DF,
+        platemap=str(csv_path),
+        join_on=["Metadata_well_position", "Metadata_Well"],
+        platemap_sep=",",
+    )
+    pd.testing.assert_frame_equal(result_csv, expected_result)
+
+
 def test_annotate_platemap_naming():
     # Test annotate with the same column name in platemap and data.
     platemap_modified_df = PLATEMAP_DF.copy().rename(