Reply To: Simulation Volume with large Voxels count

[arjun_raf]

@Adrian The case that I am running is a simple modification of the damBreak3D case but with a custom stl of my simulation volume. The code is very similar and doesn’t deviate much from the example case.

    initialize(argc, argv);
    OstreamManager clout(std::cout, "main");
    SimulationConfig c;

    UnitConverterFromResolutionAndRelaxationTime<T, DESCRIPTOR> const converter(
    int {c.N},     // resolution: number of voxels per charPhysL
    (T)   c.latticeRelaxationTime,   // latticeRelaxationTime: relaxation time, have to be greater than 0.5!
    (T)   c.charPhysLength,     // charPhysLength: reference length of simulation geometry
    (T)   c.charPhysVel,     // charPhysVelocity: maximal/highest expected velocity during simulation in __m / s__
    (T)   c.viscosity, // physViscosity: physical kinematic viscosity in __m^2 / s__
    (T)   c.density     // physDensity: physical density in __kg / m^3__
    );
    converter.print();
    STLreader<T> simvol( "openlb_m.stl", converter.getPhysDeltaX(), 1, olb::RayMode::FastRayZ, true);
    simvol.print();
    T surface_tension_coefficient_factor = std::pow(converter.getConversionFactorTime(),2)/ (c.density * std::pow(converter.getPhysDeltaX(),3));
    clout<<"Surface: "<<surface_tension_coefficient_factor * c.surface_tension_coefficient<<std::endl;
    clout<<"Lattice Size: "<<converter.getPhysDeltaX()<<std::endl;

    IndicatorLayer3D<T> extendedDomain( simvol, converter.getPhysDeltaX() );

  // Instantiation of a cuboidDecomposition with weights
    #ifdef PARALLEL_MODE_MPI
    const int noOfCuboids = singleton::mpi().getSize();
    #else
    const int noOfCuboids = 4;
    #endif
    CuboidDecomposition3D<T> cuboidDecomposition(extendedDomain, converter.getPhysDeltaX(), noOfCuboids);
    HeuristicLoadBalancer<T> loadBalancer(cuboidDecomposition);
    SuperGeometry<T,3> superGeometry(cuboidDecomposition, loadBalancer);
    prepareGeometry( converter, superGeometry, simvol);
    SuperLattice<T, DESCRIPTOR> sLattice(superGeometry);

    clout << "Overlap: " << sLattice.getOverlap() << std::endl;
    prepareLattice( converter, sLattice, superGeometry, c);
    FreeSurface3DSetup<T,DESCRIPTOR> free_surface_setup{sLattice};
    free_surface_setup.addPostProcessor();
      // Set variables from freeSurfaceHelpers.h
    sLattice.setParameter<FreeSurface::DROP_ISOLATED_CELLS>(true);
    sLattice.setParameter<FreeSurface::TRANSITION>(c.transitionThreshold);
    sLattice.setParameter<FreeSurface::LONELY_THRESHOLD>(c.lonelyThreshold);
    sLattice.setParameter<FreeSurface::HAS_SURFACE_TENSION>(c.has_surface_tension);
    sLattice.setParameter<FreeSurface::SURFACE_TENSION_PARAMETER>(surface_tension_coefficient_factor * c.surface_tension_coefficient);

    clout << "Starting Simulation ..." << std::endl;
    util::Timer<T> timer( converter.getLatticeTime( c.physTime ), superGeometry.getStatistics().getNvoxel() );
    timer.start();
    setInitialValues(sLattice, superGeometry, converter);
    for ( std::size_t iT = 0; iT < converter.getLatticeTime( c.physTime ); ++iT ) {
      getResults( sLattice, converter, iT, superGeometry, timer );
      sLattice.collideAndStream();
    }
    timer.stop();
    clout << "Simulation Complete." << std::endl;
    return 0;

I am running the sim on my university cluster. Did a test run with 10 nodes (200 CPU cores) of Intel(R) Xeon(R) CPU E5-2630, and the code was compiled with OpenMPI enabled config.mk.