Input/output (I/O) plays a crucial role in scientific applications due to the large amount of data generated. Examples include climate simulations and computational fluid dynamics. Interfacing with storage devices can lead to performance issues because latencies are significantly higher than those within the processor. Efficient storage systems are therefore important for optimal application performance. High-performance computing (HPC) vendors are moving towards exascale systems to tackle larger problems, but efficient data movement remains a challenge. Existing storage systems are often evaluated using synthetic I/O benchmarks that do not accurately represent real scientific applications and neglect the interplay between computation, communication and I/O. To address these complexities, we propose a new benchmark called {\texttt{numio}} that accurately simulates numerical computation, communication and I/O phases in real-world applications and remains configurable like a synthetic benchmark. We also introduce a benchmarking strategy for HPC systems based on semi-synthetic benchmarks of different categories. This ensemble approach allows us to capture the interactions and dependencies between jobs, providing a more comprehensive understanding of system performance.

@inproceedings{ESBFHFSK24,
	author	 = {Anna Fuchs and Jannek Squar and Michael Kuhn},
	title	 = {{Ensemble-Based System Benchmarking for HPC}},
	year	 = {2024},
	month	 = {07},
	booktitle	 = {{23rd International Symposium on Parallel and Distributed Computing, ISPDC 2024, Chur,
      Switzerland, July 8-10, 2024}},
	editor	 = {},
	publisher	 = {IEEE},
	pages	 = {1--8},
	conference	 = {ISPDC},
	isbn	 = {979-8-3503-6919-9},
	issn	 = {2996-1483},
	doi	 = {https://doi.org/10.1109/ISPDC62236.2024.10705405},
	abstract	 = {Input/output (I/O) plays a crucial role in scientific applications due to the large amount of
      data generated. Examples include climate simulations and computational fluid dynamics.
      Interfacing with storage devices can lead to performance issues because latencies are
      significantly higher than those within the processor. Efficient storage systems are therefore
      important for optimal application performance. High-performance computing (HPC) vendors are
      moving towards exascale systems to tackle larger problems, but efficient data movement remains
      a challenge. Existing storage systems are often evaluated using synthetic I/O benchmarks that
      do not accurately represent real scientific applications and neglect the interplay between
      computation, communication and I/O. To address these complexities, we propose a new benchmark
      called {\texttt{numio}} that accurately simulates numerical computation, communication and I/O
      phases in real-world applications and remains configurable like a synthetic benchmark. We also
      introduce a benchmarking strategy for HPC systems based on semi-synthetic benchmarks of
      different categories. This ensemble approach allows us to capture the interactions and
      dependencies between jobs, providing a more comprehensive understanding of system performance.},
	url	 = {https://ieeexplore.ieee.org/document/10705405},
}