Shrey1110/metadrive ci 30693 v3

.github/workflows/tests.yaml

@@ -45,7 +45,7 @@ jobs:
         max_attempts: 3
         command: git lfs pull
     - name: Build devel
-      timeout-minutes: 1
+      timeout-minutes: 4
       run: TARGET_DIR=$STRIPPED_DIR tools/release/build_stripped.sh
     - run: ./tools/op.sh setup
     - name: Build openpilot and run checks
@@ -190,25 +190,39 @@ jobs:
 
   simulator_driving:
     name: simulator driving
-    runs-on: ${{
-      (github.repository == 'commaai/openpilot') &&
-      ((github.event_name != 'pull_request') ||
-      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16"]')
-      || fromJSON('["ubuntu-24.04"]') }}
-    if: false  # FIXME: Started to timeout recently
+    runs-on: ubuntu-24.04
     steps:
     - uses: actions/checkout@v7
       with:
         submodules: true
     - run: ./tools/op.sh setup
+    - name: Install MetaDrive
+      run: |
+        . .venv/bin/activate
+        uv pip install "metadrive-simulator @ git+https://github.com/commaai/metadrive.git@minimal"
     - name: Build openpilot
       run: scons
     - name: Driving test
-      timeout-minutes: 2
+      timeout-minutes: 15
+      env:
+        LOG_ROOT: ${{ github.workspace }}/metadrive-sim-logs
+        SIM_LOGS: 1
+        SIM_FAKE_MODELD: 1
+        METADRIVE_NO_MSAA: 1
+        METADRIVE_NO_SHADOWS: 1
+        METADRIVE_NO_TERRAIN: 1
+        METADRIVE_RENDER_SCALE: "0.5"
       run: |
+        mkdir -p "$LOG_ROOT"
         source openpilot/selfdrive/test/setup_xvfb.sh
-        pytest -s openpilot/tools/sim/tests/test_metadrive_bridge.py
+        pytest -s openpilot/tools/sim/tests/test_metadrive_bridge.py --test_duration=60
+    - name: Upload simulator logs
+      if: always()
+      uses: actions/upload-artifact@v7
+      with:
+        name: metadrive-sim-logs-${{ github.run_id }}-${{ github.run_attempt }}
+        path: ${{ github.workspace }}/metadrive-sim-logs
+        retention-days: 7
 
   create_ui_report:
     name: Create UI Report

openpilot/tools/sim/bridge/metadrive/ci_render_patches.py

@@ -0,0 +1,77 @@
+"""Env-gated render simplifications for running the MetaDrive bridge on CI.
+
+Free GitHub Actions runners have no GPU; Mesa LLVMpipe software rendering
+cannot sustain the default MetaDrive pipeline in real time. These patches are
+only enabled by environment variables in the simulator CI job.
+"""
+
+import os
+
+
+def apply_ci_render_patches():
+  if os.environ.get("METADRIVE_NO_MSAA"):
+    # MetaDrive/Panda3D enables multisampling before engine startup. Prc
+    # settings are last-write-wins, so this must run before MetaDriveEnv().
+    from panda3d.core import loadPrcFileData
+    loadPrcFileData("", "framebuffer-multisample 0")
+    loadPrcFileData("", "multisamples 0")
+
+  if os.environ.get("METADRIVE_NO_SHADOWS"):
+    from metadrive.engine.core.pssm import PSSM
+
+    pssm_init_orig = PSSM.init
+
+    def pssm_init_no_render(self):
+      pssm_init_orig(self)
+      self.buffer.set_active(False)
+      self.use_pssm = False
+      self.engine.render.set_shader_inputs(use_pssm=False)
+
+    PSSM.init = pssm_init_no_render
+
+  if os.environ.get("METADRIVE_FLAT_TERRAIN_CARD"):
+    # The procedural PG-map terrain used by this test is flat for physics. The
+    # default visual path uses ShaderTerrainMesh and many fragment texture taps,
+    # which dominates LLVMpipe. Render the same road/lane attribute texture on a
+    # single flat card instead.
+    from metadrive.constants import CameraTagStateKey, CamMask
+    from metadrive.engine.core.terrain import Terrain
+    from panda3d.core import Geom, GeomNode, GeomTriangles, GeomVertexData, GeomVertexFormat, GeomVertexWriter, Shader
+
+    def _generate_ci_card(self, size, heightfield, attribute_tex, target_triangle_width=10, engine=None):
+      engine = engine or self.engine
+
+      vdata = GeomVertexData("terrain_card", GeomVertexFormat.getV3t2(), Geom.UHStatic)
+      vdata.setNumRows(4)
+      vw = GeomVertexWriter(vdata, "vertex")
+      uw = GeomVertexWriter(vdata, "texcoord")
+      for x, y in ((0, 0), (1, 0), (1, 1), (0, 1)):
+        vw.addData3(x, y, 0)
+        uw.addData2(x, y)
+
+      tris = GeomTriangles(Geom.UHStatic)
+      tris.addVertices(0, 1, 2)
+      tris.addVertices(0, 2, 3)
+
+      geom = Geom(vdata)
+      geom.addPrimitive(tris)
+      node = GeomNode("terrain_card")
+      node.addGeom(geom)
+
+      self._mesh_terrain = self.origin.attach_new_node(node)
+      self._mesh_terrain.setTwoSided(True)
+      self._mesh_terrain.hide(CamMask.MainCam)
+
+      here = os.path.dirname(os.path.abspath(__file__))
+      self._mesh_terrain.set_shader(Shader.load(Shader.SL_GLSL,
+                                                os.path.join(here, "terrain_card.vert.glsl"),
+                                                os.path.join(here, "terrain_ci.frag.glsl")))
+      self._mesh_terrain.setTag(CameraTagStateKey.Semantic, self.SEMANTIC_LABEL)
+      self._mesh_terrain.setTag(CameraTagStateKey.RGB, self.SEMANTIC_LABEL)
+      self._mesh_terrain.setTag(CameraTagStateKey.Depth, self.SEMANTIC_LABEL)
+      self._terrain_shader_set = False
+      self._set_terrain_shader(engine, attribute_tex)
+      self._mesh_terrain.set_scale(size, size, 1)
+      self._mesh_terrain.set_pos(-size / 2, -size / 2, 0)
+
+    Terrain._generate_mesh_vis_terrain = _generate_ci_card

openpilot/tools/sim/bridge/metadrive/metadrive_bridge.py

@@ -1,4 +1,5 @@
 import math
+import os
 from multiprocessing import Queue
 
 from metadrive.component.sensors.base_camera import _cuda_enable
@@ -28,6 +29,17 @@ def curve_block(length, angle=45, direction=0):
   }
 
 def create_map(track_size=60):
+  if os.environ.get("SIM_FAKE_MODELD") or os.environ.get("METADRIVE_STRAIGHT_ROAD"):
+    return dict(
+      type=MapGenerateMethod.PG_MAP_FILE,
+      lane_num=2,
+      lane_width=4.5,
+      config=[
+        None,
+        straight_block(2000),
+      ]
+    )
+
   curve_len = track_size * 2
   return dict(
     type=MapGenerateMethod.PG_MAP_FILE,
@@ -58,12 +70,18 @@ def __init__(self, dual_camera, high_quality, test_duration=math.inf, test_run=F
     self.test_duration = test_duration if self.test_run else math.inf
 
   def spawn_world(self, queue: Queue):
+    # Free GitHub Actions runners use software rendering. Render at a lower
+    # resolution in CI and upscale back to the camera size in metadrive_process.
+    render_scale = float(os.environ.get("METADRIVE_RENDER_SCALE", "1"))
+    rw = max(1, round(W * render_scale))
+    rh = max(1, round(H * render_scale))
+
     sensors = {
-      "rgb_road": (RGBCameraRoad, W, H, )
+      "rgb_road": (RGBCameraRoad, rw, rh, )
     }
 
     if self.dual_camera:
-      sensors["rgb_wide"] = (RGBCameraWide, W, H)
+      sensors["rgb_wide"] = (RGBCameraWide, rw, rh)
 
     config = dict(
       use_render=self.should_render,
@@ -87,7 +105,8 @@ def spawn_world(self, queue: Queue):
       physics_world_step_size=self.TICKS_PER_FRAME/100,
       preload_models=False,
       show_logo=False,
-      anisotropic_filtering=False
+      anisotropic_filtering=False,
+      show_terrain=not bool(os.environ.get("METADRIVE_NO_TERRAIN")),
     )
 
     return MetaDriveWorld(queue, config, self.test_duration, self.test_run, self.dual_camera)

openpilot/tools/sim/bridge/metadrive/metadrive_process.py

@@ -51,6 +51,9 @@ def arrive_destination_patch(self, *args, **kwargs):
 def metadrive_process(dual_camera: bool, config: dict, camera_array, wide_camera_array, image_lock,
                       controls_recv: Connection, simulation_state_send: Connection, vehicle_state_send: Connection,
                       exit_event, op_engaged, test_duration, test_run):
+  from openpilot.tools.sim.bridge.metadrive.ci_render_patches import apply_ci_render_patches
+  apply_ci_render_patches()
+
   arrive_dest_done = config.pop("arrive_dest_done", True)
   apply_metadrive_patches(arrive_dest_done)
 
@@ -91,6 +94,9 @@ def get_cam_as_rgb(cam):
     img = cam.perceive(to_float=False)
     if not isinstance(img, np.ndarray):
       img = img.get() # convert cupy array to numpy
+    if img.shape[0] != H or img.shape[1] != W:
+      assert H % img.shape[0] == 0 and W % img.shape[1] == 0
+      img = img.repeat(H // img.shape[0], axis=0).repeat(W // img.shape[1], axis=1)
     return img
 
   rk = Ratekeeper(100, None)

openpilot/tools/sim/bridge/metadrive/terrain_card.vert.glsl

@@ -0,0 +1,18 @@
+#version 330
+
+uniform mat4 p3d_ModelViewProjectionMatrix;
+uniform mat4 p3d_ModelMatrix;
+
+in vec4 p3d_Vertex;
+in vec2 p3d_MultiTexCoord0;
+
+out vec2 terrain_uv;
+out vec3 vtx_pos;
+out vec4 projecteds[1];
+
+void main() {
+  terrain_uv = p3d_MultiTexCoord0;
+  vtx_pos = (p3d_ModelMatrix * p3d_Vertex).xyz;
+  projecteds[0] = vec4(0.0);
+  gl_Position = p3d_ModelViewProjectionMatrix * p3d_Vertex;
+}

openpilot/tools/sim/bridge/metadrive/terrain_ci.frag.glsl

@@ -0,0 +1,52 @@
+#version 330
+
+uniform sampler2D yellow_tex;
+uniform sampler2D white_tex;
+uniform sampler2D road_tex;
+uniform sampler2D crosswalk_tex;
+uniform sampler2D grass_tex;
+uniform float grass_tex_ratio;
+uniform sampler2D attribute_tex;
+uniform float elevation_texture_ratio;
+
+in vec2 terrain_uv;
+in vec3 vtx_pos;
+in vec4 projecteds[1];
+
+out vec4 color;
+
+void main() {
+  float road_tex_ratio = 128.0;
+  float r_min = (1.0 - 1.0 / elevation_texture_ratio) / 2.0;
+  float r_max = r_min + 1.0 / elevation_texture_ratio;
+
+  vec4 attri;
+  if (abs(elevation_texture_ratio - 1.0) < 0.001) {
+    attri = texture(attribute_tex, terrain_uv);
+  } else {
+    attri = texture(attribute_tex, terrain_uv * elevation_texture_ratio + 0.5);
+  }
+
+  vec3 diffuse;
+  if ((attri.r > 0.01) && terrain_uv.x >= r_min && terrain_uv.y >= r_min && terrain_uv.x <= r_max && terrain_uv.y <= r_max) {
+    float value = attri.r;
+    if (value < 0.11) {
+      diffuse = texture(yellow_tex, terrain_uv * road_tex_ratio).rgb;
+    } else if (value < 0.21) {
+      diffuse = texture(road_tex, terrain_uv * road_tex_ratio).rgb;
+    } else if (value < 0.31) {
+      diffuse = texture(white_tex, terrain_uv * road_tex_ratio).rgb;
+    } else if (value > 0.3999 && value < 0.760001) {
+      float theta = (value - 0.39999) * 1000.0 / 180.0 * 3.1415926535;
+      vec2 uv2 = vec2(cos(theta) * terrain_uv.x - sin(theta) * terrain_uv.y,
+                      sin(theta) * terrain_uv.x + cos(theta) * terrain_uv.y);
+      diffuse = texture(crosswalk_tex, uv2 * road_tex_ratio).rgb;
+    } else {
+      diffuse = texture(white_tex, terrain_uv * road_tex_ratio).rgb;
+    }
+  } else {
+    diffuse = texture(grass_tex, terrain_uv * grass_tex_ratio * 4.0).rgb;
+  }
+
+  color = vec4(diffuse * 0.85, 1.0);
+}

openpilot/tools/sim/launch_openpilot.sh

@@ -6,10 +6,20 @@ export SIMULATION="1"
 export SKIP_FW_QUERY="1"
 export FINGERPRINT="HONDA_CIVIC_2022"
 
-export BLOCK="${BLOCK},camerad,loggerd,encoderd,micd,logmessaged,manage_athenad"
+block_list="camerad,micd,logmessaged,manage_athenad"
+
+if [[ -z "$SIM_LOGS" ]]; then
+  block_list="$block_list,loggerd,encoderd"
+fi
+
+if [[ "$SIM_FAKE_MODELD" ]]; then
+  block_list="$block_list,modeld,locationd"
+fi
+
+export BLOCK="${BLOCK},${block_list}"
 if [[ "$CI" ]]; then
   # TODO: offscreen UI should work
-  export BLOCK="${BLOCK},ui"
+  export BLOCK="${BLOCK},ui,soundd"
 fi
 
 python3 -c "from openpilot.selfdrive.test.helpers import set_params_enabled; set_params_enabled()"