YAC-Coupling 2: MPI-YAC-Compatibility 3:

examples/bubble3d/bubble3d_inputfiles.py

@@ -139,13 +139,13 @@ def main(
     zgrid = get_zgrid(icon_grid_file, num_vertical_levels)  # [m]
     xgrid = [
         0,
-        30000,
+        100000,
         2500,
     ]  # evenly spaced xhalf coords [m] # distance must match longitude in config file
     ygrid = [
         0,
-        6250,
-        1250,
+        20000,
+        4000,
     ]  # evenly spaced xhalf coords [m] # distance must match latitudes in config file
 
     ### --- settings for initial superdroplets --- ###

examples/bubble3d/src/config/bubble3d_config.yaml

@@ -34,35 +34,35 @@ kokkos_settings:
 ### SDM Runtime Parameters ###
 domain:
   nspacedims : 3                                          # no. of spatial dimensions to model
-  ngbxs : 1440                                            # total number of Gbxs
-  maxnsupers: 1440                                        # maximum number of SDs
+  ngbxs : 4800                                            # total number of Gbxs
+  maxnsupers: 4800                                        # maximum number of SDs
 
 timesteps:
   CONDTSTEP : 2                                           # time between SD condensation [s]
   COLLTSTEP : 2                                           # time between SD collision [s]
   MOTIONTSTEP : 3                                         # time between SDM motion [s]
-  COUPLTSTEP : 60                                         # time between dynamic couplings [s]
-  OBSTSTEP : 60                                           # time between SDM observations [s]
+  COUPLTSTEP : 30                                         # time between dynamic couplings [s]
+  OBSTSTEP : 30                                           # time between SDM observations [s]
   T_END : 7200                                            # time span of integration from 0s to T_END [s]
 
 ### Initialisation Parameters ###
 inputfiles:
-  constants_filename : ./libs/cleoconstants.hpp          # name of file for values of physical constants
-  grid_filename : ./build/share/bubble3d_dimlessGBxboundaries.dat   # binary filename for initialisation of GBxs / GbxMaps
+  constants_filename : /home/k/k202203/CLEO/libs/cleoconstants.hpp          # name of file for values of physical constants
+  grid_filename : /home/k/k202203/CLEO/build/share/bubble3d_dimlessGBxboundaries.dat   # binary filename for initialisation of GBxs / GbxMaps
 
 initsupers:
   type: frombinary                                        # type of initialisation of super-droplets
-  initsupers_filename : ./build/share/bubble3d_dimlessSDsinit.dat   # binary filename for initialisation of SDs
+  initsupers_filename : /home/k/k202203/CLEO/build/share/bubble3d_dimlessSDsinit.dat   # binary filename for initialisation of SDs
 
 ### Output Parameters ###
 outputdata:
-  setup_filename : ./build/bin/bubble3d_setup.txt    # .txt filename to copy configuration to
-  zarrbasedir : ./build/bin/bubble3d_sol.zarr        # zarr store base directory
+  setup_filename : /home/k/k202203/CLEO/build/bin/bubble3d_setup.txt    # .txt filename to copy configuration to
+  zarrbasedir : /home/k/k202203/CLEO/build/bin/bubble3d_sol.zarr        # zarr store base directory
   maxchunk : 2500000                                          # maximum no. of elements in chunks of zarr store array
 
 coupled_dynamics:
   type: yac
-  lower_longitude: -0.989601687                               # must match xgrid domain delta_x
-  upper_longitude: 0.895353906
-  lower_latitude: -0.392699082                                # must match ygrid domain delta_y
-  upper_latitude: 0.392699082
+  lower_longitude: -3.29867229                                # must match xgrid domain delta_x
+  upper_longitude: 2.98451302
+  lower_latitude: -1.25665                                    # must match ygrid domain delta_y
+  upper_latitude: 1.25665

examples/bubble3d/src/main_bubble3d.cpp

@@ -41,11 +41,14 @@
 #include "initialise/timesteps.hpp"
 #include "observers/collect_data_for_simple_dataset.hpp"
 #include "observers/gbxindex_observer.hpp"
+#include "observers/massmoments_observer.hpp"
 #include "observers/observers.hpp"
+#include "observers/sdmmonitor/monitor_precipitation_observer.hpp"
 #include "observers/state_observer.hpp"
 #include "observers/streamout_observer.hpp"
 #include "observers/superdrops_observer.hpp"
 #include "observers/time_observer.hpp"
+#include "observers/totnsupers_observer.hpp"
 #include "runcleo/coupleddynamics.hpp"
 #include "runcleo/couplingcomms.hpp"
 #include "runcleo/runcleo.hpp"
@@ -62,8 +65,8 @@ inline CoupledDynamics auto create_coupldyn(const Config &config, const Cartesia
   const std::array<size_t, 3> ndims({h_ndims(0), h_ndims(1), h_ndims(2)});
 
   const auto nsteps = (unsigned int)(std::ceil(t_end / couplstep) + 1);
-
-  return YacDynamics(config, couplstep, ndims, nsteps);
+  const CartesianDecomposition& decomp = gbxmaps.get_domain_decomposition();
+  return YacDynamics(config, couplstep, ndims, nsteps, decomp);
 }
 
 template <GridboxMaps GbxMaps>
@@ -84,11 +87,17 @@ inline MicrophysicalProcess auto create_microphysics(const Config &config,
                                                      const Timesteps &tsteps) {
   return NullMicrophysicalProcess{};
 }
+// inline auto create_movement(const unsigned int motionstep, const CartesianMaps &gbxmaps) {
+//   const Motion<CartesianMaps> auto motion = NullMotion{};
+//   const BoundaryConditions<CartesianMaps> auto boundary_conditions = NullBoundaryConditions{};
+
+//  return cartesian_movement(gbxmaps, motion, boundary_conditions);
+// }
 
 inline auto create_movement(const unsigned int motionstep, const CartesianMaps &gbxmaps) {
   const auto terminalv = NullTerminalVelocity{};
   const Motion<CartesianMaps> auto motion =
-      CartesianMotion(motionstep, &step2dimlesstime, terminalv);
+       CartesianMotion(motionstep, &step2dimlesstime, terminalv);
 
   const BoundaryConditions<CartesianMaps> auto boundary_conditions = NullBoundaryConditions{};
 
@@ -127,9 +136,17 @@ inline Observer auto create_observer(const Config &config, const Timesteps &tste
 
   const Observer auto obs3 = StateObserver(obsstep, dataset, maxchunk, ngbxs);
 
+  const Observer auto obs4 = MassMomentsObserver(obsstep, dataset, store, maxchunk, ngbxs);
+
+  const Observer auto obs5 = MassMomentsRaindropsObserver(obsstep, dataset, store, maxchunk, ngbxs);
+
+  const Observer auto obs6 = MonitorPrecipitationObserver(obsstep, dataset, store, maxchunk, ngbxs);
+
+  const Observer auto obs7 = TotNsupersObserver(obsstep, dataset, store, maxchunk);
+
   const Observer auto obssd = create_superdrops_observer(obsstep, dataset, store, maxchunk);
 
-  return obssd >> obs3 >> obs2 >> obs1 >> obs0;
+  return obssd >> obs7 >> obs6 >> obs5 >> obs4 >> obs3 >> obs2 >> obs1 >> obs0;
 }
 
 template <typename Dataset, typename Store>

libs/cartesiandomain/cartesian_decomposition.cpp

@@ -37,6 +37,13 @@ std::array<size_t, 3> CartesianDecomposition::get_local_partition_size() const {
   return partition_sizes[my_rank];
 };
 
+std::vector<std::vector<double>> CartesianDecomposition::get_local_gridbox_bounds() const {
+  return gridbox_bounds;
+};
+std::array<std::array<double, 3>, 2> CartesianDecomposition::get_domain_bounds() const {
+  return domain_bounds;
+};
+
 void CartesianDecomposition::set_gridbox_bounds(GbxBoundsFromBinary gfb) {
   // Function to store gridbox bounds in an array
   // Array contains lower (i) and upper bounds (i+1) of all the local gridboxes
@@ -394,7 +401,7 @@ int find_best_decomposition(std::vector<std::vector<size_t>> &factors,
                             const std::vector<size_t> ndims) {
   std::array<size_t, 3> partition_origin, partition_size;
   int comm_size, best_factorization = -1;
-  double vertical_split_penalization = 1.0;
+  double vertical_split_penalization = 100.0;
   comm_size = init_communicator::get_comm_size();
 
   // Calculates the ideal (most even possible) division and initializes the minimum error

libs/cartesiandomain/cartesian_decomposition.hpp

@@ -93,6 +93,9 @@ class CartesianDecomposition {
   std::array<size_t, 3> get_local_partition_origin() const;
   std::array<size_t, 3> get_local_partition_size() const;
 
+  std::vector<std::vector<double>> get_local_gridbox_bounds() const;
+  std::array<std::array<double, 3>, 2> get_domain_bounds() const;
+
   // Get partition index and partition coordinates
   int get_partition_index_from_slice(std::array<int, 3> slice_indices) const;
   std::array<int, 3> get_slice_indices_from_partition(int partition_index) const;