@inproceedings{71cca4550db64635b777d5d09cb66780,
title = "A statistics-based performance testing methodology for cloud applications",
abstract = "The low cost of resource ownership and flexibility have led users to increasingly port their applications to the clouds. To fully realize the cost benefits of cloud services, users usually need to reliably know the execution performance of their applications. However, due to the random performance fluctuations experienced by cloud applications, the black box nature of public clouds and the cloud usage costs, testing on clouds to acquire accurate performance results is extremely difficult. In this paper, we present a novel cloud performance testing methodology called PT4Cloud. By employing non-parametric statistical approaches of likelihood theory and the bootstrap method, PT4Cloud provides reliable stop conditions to obtain highly accurate performance distributions with confidence bands. These statistical approaches also allow users to specify intuitive accuracy goals and easily trade between accuracy and testing cost. We evaluated PT4Cloud with 33 benchmark configurations on Amazon Web Service and Chameleon clouds. When compared with performance data obtained from extensive performance tests, PT4Cloud provides testing results with 95.4% accuracy on average while reducing the number of test runs by 62%. We also propose two test execution reduction techniques for PT4Cloud, which can reduce the number of test runs by 90.1% while retaining an average accuracy of 91%. We compared our technique to three other techniques and found that our results are much more accurate.",
keywords = "Cloud computing, Non- parametric statistics, Performance testing, Resource contention",
author = "Sen He and Glenna Manns and John Saunders and Wei Wang and Lori Pollock and Soffa, {Mary Lou}",
note = "Publisher Copyright: {\textcopyright} 2019 ACM.; 27th ACM Joint European Software Engineering Conference and Symposium on the Foundations of Software Engineering, ESEC/FSE 2019 ; Conference date: 26-08-2019 Through 30-08-2019",
year = "2019",
month = aug,
day = "12",
doi = "10.1145/3338906.3338912",
language = "English (US)",
series = "ESEC/FSE 2019 - Proceedings of the 2019 27th ACM Joint Meeting European Software Engineering Conference and Symposium on the Foundations of Software Engineering",
publisher = "Association for Computing Machinery, Inc",
pages = "188--199",
editor = "Sven Apel and Marlon Dumas and Alessandra Russo and Dietmar Pfahl",
booktitle = "ESEC/FSE 2019 - Proceedings of the 2019 27th ACM Joint Meeting European Software Engineering Conference and Symposium on the Foundations of Software Engineering",
}