fix: mirror dependencies declared by wheels built from sdists (#5202)

Author: OliverBryant

xinference/deploy/docker/pypiserver/download_packages.py

@@ -411,31 +411,78 @@ def main() -> None:
     # 4. Build wheels for sdist-only downloads so the runtime never
     #    compiles. Failures keep the sdist (the runtime image has a
     #    toolchain) and are recorded in the report.
+    #
+    #    A built wheel can declare dependencies that the sdist's static
+    #    metadata omitted (e.g. GPTQModel publishes an sdist whose
+    #    PKG-INFO lists no runtime requirements, while the wheel its
+    #    setup.py builds does). The locks above only saw the sdist
+    #    metadata, so fetch each built wheel's dependencies explicitly
+    #    and repeat until no new sdists arrive.
     # ------------------------------------------------------------------
-    for sdist in sorted(dest.iterdir()):
-        if sdist.suffix not in (".gz", ".zip", ".bz2") and not sdist.name.endswith(
-            ".tar.gz"
-        ):
-            continue
-        proc = run(
-            [
-                sys.executable,
-                "-m",
-                "pip",
-                "wheel",
-                "--quiet",
-                "--no-deps",
-                "--wheel-dir",
-                str(dest),
-                str(sdist),
-            ],
-            env=package_build_env,
-        )
-        if proc.returncode == 0:
-            sdist.unlink()
-        else:
-            print(f"WARN: keeping sdist {sdist.name}", flush=True)
-            sdist_left.append(sdist.name)
+    failed_sdists: Set[str] = set()
+    while True:
+        built_wheels: List[Path] = []
+        for sdist in sorted(dest.iterdir()):
+            if sdist.suffix not in (".gz", ".zip", ".bz2") and not sdist.name.endswith(
+                ".tar.gz"
+            ):
+                continue
+            if sdist.name in failed_sdists:
+                continue
+            wheels_before = {p.name for p in dest.glob("*.whl")}
+            proc = run(
+                [
+                    sys.executable,
+                    "-m",
+                    "pip",
+                    "wheel",
+                    "--quiet",
+                    "--no-deps",
+                    "--wheel-dir",
+                    str(dest),
+                    str(sdist),
+                ],
+                env=package_build_env,
+            )
+            if proc.returncode == 0:
+                sdist.unlink()
+                built_wheels.extend(
+                    p for p in dest.glob("*.whl") if p.name not in wheels_before
+                )
+            else:
+                print(f"WARN: keeping sdist {sdist.name}", flush=True)
+                failed_sdists.add(sdist.name)
+        if not built_wheels:
+            break
+        for wheel in built_wheels:
+            proc = pip_download(
+                [str(wheel)],
+                dest,
+                index_url=args.index_url,
+                extra_index_urls=[args.pytorch_index],
+                constraints=master_constraints,
+                env=package_build_env,
+            )
+            if proc.returncode != 0:
+                print(
+                    f"WARN: retrying dependencies of '{wheel.name}' "
+                    "without constraints",
+                    flush=True,
+                )
+                proc = pip_download(
+                    [str(wheel)],
+                    dest,
+                    index_url=args.index_url,
+                    extra_index_urls=[args.pytorch_index],
+                    env=package_build_env,
+                )
+                if proc.returncode != 0:
+                    sys.exit(
+                        f"FATAL: failed to fetch dependencies of built "
+                        f"wheel '{wheel.name}'"
+                    )
+                unconstrained_fallbacks.append(wheel.name)
+    sdist_left.extend(sorted(failed_sdists))
 
     # ------------------------------------------------------------------
     # Report.

xinference/deploy/docker/pypiserver/tests/test_scripts.py

@@ -417,10 +417,18 @@ def fake_pip_download(args, target, **kwargs):
         download_calls.append((list(args), kwargs))
         if args == ["model-pin==1.0"]:
             (target / "source-1.0.tar.gz").write_text("sdist")
+        # The wheel built from the sdist declares a dependency that the
+        # sdist metadata omitted; it only exists as an sdist itself.
+        if args == [str(dest / "source-1.0-py3-none-any.whl")]:
+            (target / "hiddendep-1.0.tar.gz").write_text("sdist")
         return subprocess.CompletedProcess([], 0)
 
     def fake_run(cmd, **kwargs):
         run_calls.append((list(cmd), kwargs))
+        if "wheel" in cmd:
+            sdist_name = Path(cmd[-1]).name
+            wheel_name = sdist_name.replace(".tar.gz", "-py3-none-any.whl")
+            (dest / wheel_name).write_text("wheel")
         return subprocess.CompletedProcess(cmd, 0)
 
     monkeypatch.setattr(downloader, "uv_compile", fake_uv_compile)
@@ -458,6 +466,15 @@ def fake_run(cmd, **kwargs):
     assert all(call[1]["env"]["TORCH_VERSION"] == "2.12.3" for call in download_calls)
     assert all(call[1]["env"]["TORCH_VERSION"] == "2.12.3" for call in run_calls)
     assert not (dest / "source-1.0.tar.gz").exists()
+    # Dependencies of wheels built from sdists are fetched from the built
+    # wheel's own metadata, transitively until no new sdists arrive.
+    assert [str(dest / "source-1.0-py3-none-any.whl")] in [
+        call[0] for call in download_calls
+    ]
+    assert [str(dest / "hiddendep-1.0-py3-none-any.whl")] in [
+        call[0] for call in download_calls
+    ]
+    assert not (dest / "hiddendep-1.0.tar.gz").exists()
     report = json.loads((manifest / "report.json").read_text())
     assert "transformers==5.13.1" in report["runtime_constraints"]
     assert report["unconstrained_fallbacks"] == []