Scientific applications generate massive amounts of data, posing storage limitations and network traffic challenges. While scientists struggle with the usage of application-side compression and parallel I/O, we design a transparent feature. Our Lustre-based prototype automatically applies lossless compression, offering flexibility in compression-related decisions to minimize computational costs and optimize application performance. We outline the challenges posed by our prototype and illustrate through a comprehensive assessment how integrating Lustre and ZFS as backend solutions provides the essential elements for performance and scalability: specifically, asynchronous operations and parallel processing of compression. In our evaluation, we illustrate the interaction of different buffer levels within a distributed system. Additionally, we showcase how the I/O pattern, hardware setup, and various system software optimizations can impact overall performance and influence the choice of compression strategy.

@inproceedings{TECILFSK24,
	author	 = {Anna Fuchs and Jannek Squar and Michael Kuhn},
	title	 = {{Towards End-to-End Compression in Lustre}},
	year	 = {2024},
	month	 = {07},
	booktitle	 = {{23rd International Symposium on Parallel and Distributed Computing, ISPDC 2024, Chur,
      Switzerland, July 8-10, 2024}},
	editor	 = {},
	publisher	 = {IEEE},
	pages	 = {1--8},
	conference	 = {ISPDC},
	isbn	 = {979-8-3503-6919-9},
	issn	 = {2996-1483},
	doi	 = {https://doi.org/10.1109/ISPDC62236.2024.10705396},
	abstract	 = {Scientific applications generate massive amounts of data, posing storage limitations and
      network traffic challenges. While scientists struggle with the usage of application-side
      compression and parallel I/O, we design a transparent feature. Our Lustre-based prototype
      automatically applies lossless compression, offering flexibility in compression-related
      decisions to minimize computational costs and optimize application performance. We outline the
      challenges posed by our prototype and illustrate through a comprehensive assessment how
      integrating Lustre and ZFS as backend solutions provides the essential elements for
      performance and scalability: specifically, asynchronous operations and parallel processing of
      compression. In our evaluation, we illustrate the interaction of different buffer levels
      within a distributed system. Additionally, we showcase how the I/O pattern, hardware setup,
      and various system software optimizations can impact overall performance and influence the
      choice of compression strategy.},
	url	 = {https://ieeexplore.ieee.org/document/10705396},
}