Heterogeneous clusters are a widely utilized class of supercomputers assembled from different types of computing devices, for instance CPUs and GPUs, providing a huge computational potential. Programming them in a scalable way exploiting the maximal performance introduces numerous challenges such as optimizations for different computing devices, dealing with multiple levels of parallelism, the application of different programming models, work distribution, and hiding of communication with computation. We utilize the lattice Boltzmann method for fluid flow as a representative of a scientific computing application and develop a holistic implementation for large-scale CPU/GPU heterogeneous clusters. We review and combine a set of best practices and techniques ranging from optimizations for the particular computing devices to the orchestration of tens of thousands of CPU cores and thousands of GPUs. Eventually, we come up with an implementation using all the available computational resources for the lattice Boltzmann method operators. Our approach shows excellent scalability behavior making it future-proof for heterogeneous clusters of the upcoming architectures on the exaFLOPS scale. Parallel efficiencies of more than 90% are achieved leading to 2604.72 GLUPS utilizing 24576 CPU cores and 2048 GPUs of the CPU/GPU heterogeneous cluster Piz Daint and computing more than 6.8×10^9 lattice cells.

@article{AHSIAOTLBM17,
	author	 = {Christoph Riesinger and Arash Bakhtiari and Martin Schreiber and Philipp Neumann and Hans-Joachim Bungartz},
	title	 = {{A holistic scalable implementation approach of the lattice Boltzmann method for CPU/GPU
      heterogeneous clusters}},
	year	 = {2017},
	month	 = {11},
	editor	 = {Karlheinz Schwarz},
	publisher	 = {MDPI},
	journal	 = {Computation},
	series	 = {5(4)},
	pages	 = {48},
	issn	 = {2079-3197},
	doi	 = {http://dx.doi.org/10.3390/computation5040048},
	abstract	 = {Heterogeneous clusters are a widely utilized class of supercomputers assembled from
      different types of computing devices, for instance CPUs and GPUs, providing a huge
      computational potential. Programming them in a scalable way exploiting the maximal performance
      introduces numerous challenges such as optimizations for different computing devices, dealing
      with multiple levels of parallelism, the application of different programming models, work
      distribution, and hiding of communication with computation. We utilize the lattice Boltzmann
      method for fluid flow as a representative of a scientific computing application and develop a
      holistic implementation for large-scale CPU/GPU heterogeneous clusters. We review and combine
      a set of best practices and techniques ranging from optimizations for the particular computing
      devices to the orchestration of tens of thousands of CPU cores and thousands of GPUs.
      Eventually, we come up with an implementation using all the available computational resources
      for the lattice Boltzmann method operators. Our approach shows excellent scalability behavior
      making it future-proof for heterogeneous clusters of the upcoming architectures on the
      exaFLOPS scale. Parallel efficiencies of more than 90\% are achieved leading to 2604.72 GLUPS
      utilizing 24576 CPU cores and 2048 GPUs of the CPU/GPU heterogeneous cluster Piz Daint and
      computing more than 6.8x10^9 lattice cells.},
}