Skipping tcp tests, fixing ci (#3775)

Description:
This PR resolves the CI hangs and provides a clear guide to users on how
to migrate away from the legacy `tcp://` initialization method

Check list:

- [x] New tests are added (if a new feature is added)
- [x] New doc strings: description and/or example code are in RST format
- [ ] Documentation is updated (if required)

Author: TahaZahid05

ignite/distributed/comp_models/native.py

@@ -107,6 +107,13 @@ def _create_from_backend(
             rank: int | None = None,
             **kwargs: Any,
         ) -> None:
+            if init_method is not None and init_method.startswith("tcp://"):
+                raise ValueError(
+                    f"TCP initialization via init_method='{init_method}' will hang. "
+                    "To fix this, please configure MASTER_ADDR and MASTER_PORT in the environment and "
+                    "use 'env://' (or omit init_method)."
+                )
+
             if backend == dist.Backend.NCCL and not torch.cuda.is_available():
                 raise RuntimeError("Nccl backend is required but no cuda capable devices")
             self._backend = backend

ignite/distributed/launcher.py

@@ -132,12 +132,6 @@ def training(local_rank, config, **kwargs):
             with idist.Parallel(backend=backend, init_method='file:///d:/tmp/some_file', nproc_per_node=4) as parallel:
                 parallel.run(training, config, a=1, b=2)
 
-        Initializing the process using ``tcp://``
-
-        .. code-block:: python
-
-            with idist.Parallel(backend=backend, init_method='tcp://10.1.1.20:23456', nproc_per_node=4) as parallel:
-                parallel.run(training, config, a=1, b=2)
 
 
         3) Single node, Multi-TPU training launched with `python`
@@ -233,6 +227,13 @@ def __init__(
                 if value is not None:
                     raise ValueError(f"If backend is None, argument '{name}' should be also None, but given {value}")
 
+        if init_method is not None and init_method.startswith("tcp://"):
+            raise ValueError(
+                f"TCP initialization via init_method='{init_method}' will hang. "
+                "To fix this, please configure MASTER_ADDR and MASTER_PORT in the environment and "
+                "use 'env://' (or omit init_method)."
+            )
+
         self.backend = backend
         self._spawn_params = None
         self.init_method = init_method

tests/ignite/conftest.py

@@ -231,15 +231,19 @@ def _setup_free_port(local_rank):
 @pytest.fixture()
 def distributed_context_single_node_nccl(local_rank, world_size):
     free_port = _setup_free_port(local_rank)
+    os.environ["MASTER_ADDR"] = "localhost"
+    os.environ["MASTER_PORT"] = str(free_port)
 
     dist_info = {
         "backend": "nccl",
         "world_size": world_size,
         "rank": local_rank,
-        "init_method": f"tcp://localhost:{free_port}",
+        "init_method": "env://",
     }
     yield _create_dist_context(dist_info, local_rank)
     _destroy_dist_context()
+    os.environ.pop("MASTER_ADDR", None)
+    os.environ.pop("MASTER_PORT", None)
 
 
 @pytest.fixture()
@@ -253,8 +257,10 @@ def distributed_context_single_node_gloo(local_rank, world_size):
         init_method = f"file:///{temp_file.name.replace(backslash, '/')}"
     else:
         free_port = _setup_free_port(local_rank)
-        init_method = f"tcp://localhost:{free_port}"
+        init_method = "env://"
         temp_file = None
+        os.environ["MASTER_ADDR"] = "localhost"
+        os.environ["MASTER_PORT"] = str(free_port)
 
     dist_info = {
         "backend": "gloo",
@@ -267,6 +273,9 @@ def distributed_context_single_node_gloo(local_rank, world_size):
     _destroy_dist_context()
     if temp_file:
         temp_file.close()
+    else:
+        os.environ.pop("MASTER_ADDR", None)
+        os.environ.pop("MASTER_PORT", None)
 
 
 @pytest.fixture()
@@ -475,7 +484,9 @@ def distributed(request, local_rank, world_size):
         else:
             temp_file = None
             free_port = _setup_free_port(local_rank)
-            init_method = f"tcp://localhost:{free_port}"
+            init_method = "env://"
+            os.environ["MASTER_ADDR"] = "localhost"
+            os.environ["MASTER_PORT"] = str(free_port)
 
         dist_info = {
             "world_size": world_size,
@@ -494,6 +505,9 @@ def distributed(request, local_rank, world_size):
         _destroy_dist_context()
         if temp_file:
             temp_file.close()
+        else:
+            os.environ.pop("MASTER_ADDR", None)
+            os.environ.pop("MASTER_PORT", None)
 
     elif request.param == "horovod":
         request.node.stash[is_horovod_stash_key] = True

tests/ignite/distributed/comp_models/test_native.py

@@ -334,7 +334,8 @@ def _test__native_dist_model_create_from_context_set_local_rank(true_conf):
 def _test__native_dist_model_create_from_context_no_dist(true_backend, true_device):
     assert _NativeDistModel.create_from_context() is None
 
-    dist.init_process_group(true_backend, "tcp://0.0.0.0:2222", world_size=1, rank=0)
+    store = dist.TCPStore("0.0.0.0", 2222, world_size=1, is_master=True)
+    dist.init_process_group(true_backend, store=store, world_size=1, rank=0)
     dist.barrier()
 
     _test__native_dist_model_create_from_context_no_local_rank()
@@ -358,7 +359,9 @@ def _test__native_dist_model_create_from_context_no_dist(true_backend, true_devi
 def _test__native_dist_model_create_from_context_dist(local_rank, rank, world_size, true_backend, true_device):
     assert _NativeDistModel.create_from_context() is None
 
-    dist.init_process_group(true_backend, "tcp://0.0.0.0:2222", world_size=world_size, rank=rank)
+    is_master = rank == 0
+    store = dist.TCPStore("0.0.0.0", 2222, world_size=world_size, is_master=is_master)
+    dist.init_process_group(true_backend, store=store, world_size=world_size, rank=rank)
     dist.barrier()
     if torch.cuda.is_available():
         torch.cuda.set_device(local_rank)
@@ -397,7 +400,7 @@ def test__native_dist_model_create_no_dist_nccl(clean_env):
 
 
 @pytest.mark.distributed
-@pytest.mark.parametrize("init_method", [None, "tcp://0.0.0.0:22334", "FILE"])
+@pytest.mark.parametrize("init_method", [None, "FILE"])
 def test__native_dist_model_create_dist_gloo_1(init_method, get_fixed_dirname, local_rank, world_size):
     if init_method == "FILE":
         init_method = f"file://{get_fixed_dirname('native_dist_model_create_dist_gloo_1')}/shared"
@@ -418,7 +421,7 @@ def test__native_dist_model_create_dist_gloo_2(local_rank, world_size):
 
 @pytest.mark.distributed
 @pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Skip if no GPU")
-@pytest.mark.parametrize("init_method", [None, "tcp://0.0.0.0:22334", "FILE"])
+@pytest.mark.parametrize("init_method", [None, "FILE"])
 def test__native_dist_model_create_dist_nccl_1(init_method, get_fixed_dirname, local_rank, world_size):
     if init_method == "FILE":
         init_method = f"file://{get_fixed_dirname('native_dist_model_create_dist_nccl_1')}/shared"
@@ -444,7 +447,9 @@ def test__native_dist_model_create_dist_nccl_2(local_rank, world_size):
 def test__native_dist_model_warning_index_less_localrank(local_rank, world_size):
     assert _NativeDistModel.create_from_context() is None
 
-    dist.init_process_group("nccl", "tcp://0.0.0.0:2222", world_size=world_size, rank=local_rank)
+    is_master = local_rank == 0
+    store = dist.TCPStore("0.0.0.0", 2222, world_size=world_size, is_master=is_master)
+    dist.init_process_group("nccl", store=store, world_size=world_size, rank=local_rank)
     dist.barrier()
     # We deliberately incorrectly set cuda device to 0
     torch.cuda.set_device(0)
@@ -496,7 +501,7 @@ def _test__native_dist_model_spawn(backend, num_workers_per_machine, device, ini
 
 @pytest.mark.distributed
 @pytest.mark.skipif("WORLD_SIZE" in os.environ, reason="Skip if launched as multiproc")
-@pytest.mark.parametrize("init_method", [None, "CUSTOM_ADDR_PORT", "env://", "tcp://0.0.0.0:22334", "FILE"])
+@pytest.mark.parametrize("init_method", [None, "CUSTOM_ADDR_PORT", "env://", "FILE"])
 def test__native_dist_model_spawn_gloo(init_method, dirname):
     spawn_kwargs = {}
 
@@ -532,7 +537,7 @@ def test__native_dist_model_spawn_gloo(init_method, dirname):
 @pytest.mark.distributed
 @pytest.mark.skipif("WORLD_SIZE" in os.environ, reason="Skip if launched as multiproc")
 @pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Skip if no GPU")
-@pytest.mark.parametrize("init_method", [None, "CUSTOM_ADDR_PORT", "tcp://0.0.0.0:22334", "FILE"])
+@pytest.mark.parametrize("init_method", [None, "CUSTOM_ADDR_PORT", "FILE"])
 def test__native_dist_model_spawn_nccl(init_method, dirname):
     spawn_kwargs = {}
 
@@ -720,3 +725,8 @@ def test__setup_ddp_vars_from_slurm_env_bad_configs():
             "SLURM_JOB_ID": "12345",
         }
         _setup_ddp_vars_from_slurm_env(environ)
+
+
+def test__native_dist_model_tcp_init_method_error():
+    with pytest.raises(ValueError, match="will hang. To fix this, please configure MASTER_ADDR"):
+        _NativeDistModel.create_from_backend(backend="gloo", init_method="tcp://10.1.1.20:23456", rank=0, world_size=1)

tests/ignite/distributed/test_launcher.py

@@ -103,13 +103,6 @@ def _test_check_idist_parallel_torch_launch(init_method, fp, backend, nprocs):
     "init_method",
     [
         None,
-        pytest.param(
-            "tcp://0.0.0.0:29500",
-            marks=pytest.mark.skipif(
-                "dev" in torch.__version__,
-                reason="Skip tcp:// init_method with torchrun on nightly due to incompatibility",
-            ),
-        ),
         "FILE",
     ],
 )
@@ -241,7 +234,7 @@ def _test_func(index, ws, device, backend, true_init_method):
 @pytest.mark.distributed
 @pytest.mark.skipif("WORLD_SIZE" in os.environ, reason="Skip if launched as multiproc")
 @pytest.mark.skipif(not has_native_dist_support, reason="Skip if no native dist support")
-@pytest.mark.parametrize("init_method", ["env://", "tcp://0.0.0.0:29500", "FILE"])
+@pytest.mark.parametrize("init_method", ["env://", "FILE"])
 @pytest.mark.parametrize(
     "backend",
     ["gloo", pytest.param("nccl", marks=pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Skip if no GPU"))],
@@ -259,7 +252,7 @@ def test_idist_parallel_spawn_n_procs_native(init_method, backend, dirname):
 @pytest.mark.distributed
 @pytest.mark.skipif("WORLD_SIZE" not in os.environ, reason="Skip if not launched as multiproc")
 @pytest.mark.skipif(not has_native_dist_support, reason="Skip if no native dist support")
-@pytest.mark.parametrize("init_method", ["env://", "tcp://0.0.0.0:29500", "FILE"])
+@pytest.mark.parametrize("init_method", ["env://", "FILE"])
 @pytest.mark.parametrize(
     "backend",
     ["gloo", pytest.param("nccl", marks=pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Skip if no GPU"))],
@@ -296,3 +289,9 @@ def test_idist_parallel_spawn_params_xla():
         res = parallel._spawn_params
         assert "nproc_per_node" in res and res["nproc_per_node"] == 8
         assert "start_method" in res and res["start_method"] == "fork"
+
+
+def test_idist_parallel_tcp_init_method_error():
+    with pytest.raises(ValueError, match="will hang. To fix this, please configure MASTER_ADDR"):
+        with idist.Parallel(backend="gloo", init_method="tcp://10.1.1.20:23456", nproc_per_node=1) as parallel:
+            pass

tests/ignite/distributed/utils/test_native.py

@@ -38,7 +38,7 @@ def _test_native_distrib_single_node_launch_tool(backend, device, local_rank, wo
 
 @pytest.mark.distributed
 @pytest.mark.skipif(not has_native_dist_support, reason="Skip if no native dist support")
-@pytest.mark.parametrize("init_method", [None, "tcp://0.0.0.0:22334", "FILE"])
+@pytest.mark.parametrize("init_method", [None, "FILE"])
 def test_native_distrib_single_node_launch_tool_gloo(init_method, get_fixed_dirname, local_rank, world_size):
     from datetime import timedelta
 
@@ -56,7 +56,7 @@ def test_native_distrib_single_node_launch_tool_gloo(init_method, get_fixed_dirn
 @pytest.mark.distributed
 @pytest.mark.skipif(not has_native_dist_support, reason="Skip if no native dist support")
 @pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Skip if no GPU")
-@pytest.mark.parametrize("init_method", [None, "tcp://0.0.0.0:22334", "FILE"])
+@pytest.mark.parametrize("init_method", [None, "FILE"])
 def test_native_distrib_single_node_launch_tool_nccl(init_method, get_fixed_dirname, local_rank, world_size):
     if init_method == "FILE":
         init_method = f"file://{get_fixed_dirname('native_distrib_single_node_launch_tool_nccl')}/shared"
@@ -80,7 +80,7 @@ def _test_native_distrib_single_node_spawn(init_method, backend, device, **kwarg
 @pytest.mark.distributed
 @pytest.mark.skipif(not has_native_dist_support, reason="Skip if no native dist support")
 @pytest.mark.skipif("WORLD_SIZE" in os.environ, reason="Skip if launched as multiproc")
-@pytest.mark.parametrize("init_method", [None, "tcp://0.0.0.0:22334", "FILE"])
+@pytest.mark.parametrize("init_method", [None, "FILE"])
 def test_native_distrib_single_node_spawn_gloo(init_method, dirname):
     from datetime import timedelta
 
@@ -97,7 +97,7 @@ def test_native_distrib_single_node_spawn_gloo(init_method, dirname):
 @pytest.mark.skipif(not has_native_dist_support, reason="Skip if no native dist support")
 @pytest.mark.skipif("WORLD_SIZE" in os.environ, reason="Skip if launched as multiproc")
 @pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Skip if no GPU")
-@pytest.mark.parametrize("init_method", [None, "tcp://0.0.0.0:22334", "FILE"])
+@pytest.mark.parametrize("init_method", [None, "FILE"])
 def test_native_distrib_single_node_spawn_nccl(init_method, dirname):
     if init_method == "FILE":
         init_method = f"file://{dirname}/shared"