References.bib

@inproceedings{simmonds2018complexity,
	title={Complexity and the engineering of bug-free software},
	author={Simmonds, Devon M},
	booktitle={Proceedings of the International Conference on Frontiers in Education: Computer Science and Computer Engineering (FECS)},
	pages={94--100},
	year={2018},
	organization={The Steering Committee of The World Congress in Computer Science, Computer~…}
}

@book{weinberg2008perfect,
	author = {Weinberg, Gerald M.},
	title = {Perfect Software: And Other Illusions about Testing},
	year = {2008},
	isbn = {0932633692},
	publisher = {Dorset House Publishing Co., Inc.},
	address = {USA},
	abstract = {Everyone has a role to play in software testing -- even people outside a project team. Testers, developers, managers, customers, and users shape the process and results of testing, often unwittingly. Rather than continue to generate stacks of documents and fuel animosity, testers can cultivate rich opportunities and relationships by integrating an effective testing mentality into any process.Jerry Weinberg, author of The Psychology of Computer Programming and more than forty nonfiction books, sets out to disprove destructive notions about testing and testers in Perfect Software And Other Illusions About Testing. With a blend of wit, storytelling, and jaw-dropping insight that has won him fans around the world, Weinberg deftly separates what is expected, significant, and possible in software testing. He destroys fallacies and steers readers clear of common mistakes. We test because people are not perfect, and simply testing more does not guarantee better quality. This book guides test strategy development that's scalable for any project.Topics include: - Why Not Just Test Everything?- Information Immunity- What Makes a Test Good?- Major Fallacies About Testing- Determining Significance- Testing Without Machinery- and much more.}
}

@book{glenford2012art,
	title = {Art of Software Testing (3rd Edition)},
	author = {Myers, Glenford, J. and Badgett, Tom and Sandler, Corey},
	year = {2012},
	url = {https://app.knovel.com/hotlink/khtml/id:kt011N5M57/art-software-testing/psychology-testing},
	isbn = {978-1-118-03196-4},
	publisher = {John Wiley \& Sons},
	pages = {ix-8},
}

@online{googleScholar,
	author = {Google},
	title = "Google Scholar",
	year = 2022,
	url = {https://scholar.google.com/},
	urldate = {2022-08-02}
}

@inproceedings{aiRicca2021,
	author={Ricca, Filippo and Marchetto, Alessandro and Stocco, Andrea},
	booktitle={2021 IEEE International Conference on Software Testing, Verification and Validation Workshops (ICSTW)},
	title={AI-based Test Automation: A Grey Literature Analysis},
	year={2021},
	volume={},
	number={},
	pages={263-270},
	doi={10.1109/ICSTW52544.2021.00051}
}

@article{garousi2016when,
	title = {When and what to automate in software testing? A multi-vocal literature review},
	journal = {Information and Software Technology},
	volume = {76},
	pages = {92-117},
	year = {2016},
	issn = {0950-5849},
	doi = {https://doi.org/10.1016/j.infsof.2016.04.015},
	url = {https://www.sciencedirect.com/science/article/pii/S0950584916300702},
	author = {Vahid Garousi and Mika V. Mäntylä},
	keywords = {Software test automation, Decision support, When to automate, What to automate, Multivocal literature review, Systematic literature review, Systematic Mapping study},
	abstract = {Context
	Many organizations see software test automation as a solution to decrease testing costs and to reduce cycle time in software development. However, establishment of automated testing may fail if test automation is not applied in the right time, right context and with the appropriate approach.
	Objective
	The decisions on when and what to automate is important since wrong decisions can lead to disappointments and major wrong expenditures (resources and efforts). To support decision making on when and what to automate, researchers and practitioners have proposed various guidelines, heuristics and factors since the early days of test automation technologies. As the number of such sources has increased, it is important to systematically categorize the current state-of-the-art and -practice, and to provide a synthesized overview.
	Method
	To achieve the above objective, we have performed a Multivocal Literature Review (MLR) study on when and what to automate in software testing. A MLR is a form of a Systematic Literature Review (SLR) which includes the grey literature (e.g., blog posts and white papers) in addition to the published (formal) literature (e.g., journal and conference papers). We searched the academic literature using the Google Scholar and the grey literature using the regular Google search engine.
	Results
	Our MLR and its results are based on 78 sources, 52 of which were grey literature and 26 were formally published sources. We used the qualitative analysis (coding) to classify the factors affecting the when- and what-to-automate questions to five groups: (1) Software Under Test (SUT)-related factors, (2) test-related factors, (3) test-tool-related factors, (4) human and organizational factors, and (5) cross-cutting and other factors. The most frequent individual factors were: need for regression testing (44 sources), economic factors (43), and maturity of SUT (39).
	Conclusion
	We show that current decision-support in software test automation provides reasonable advice for industry, and as a practical outcome of this research we have summarized it as a checklist that can be used by practitioners. However, we recommend developing systematic empirically-validated decision-support approaches as the existing advice is often unsystematic and based on weak empirical evidence.}
}

@article{garousi2016software,
	title = {A systematic literature review of literature reviews in software testing},
	journal = {Information and Software Technology},
	volume = {80},
	pages = {195-216},
	year = {2016},
	issn = {0950-5849},
	doi = {https://doi.org/10.1016/j.infsof.2016.09.002},
	url = {https://www.sciencedirect.com/science/article/pii/S0950584916301446},
	author = {Vahid Garousi and Mika V. Mäntylä},
	keywords = {Secondary studies, Tertiary study, Software testing, Systematic mapping, Systematic literature reviews, Surveys},
	abstract = {Context
	Any newcomer or industrial practitioner is likely to experience difficulties in digesting large volumes of knowledge in software testing. In an ideal world, all knowledge used in industry, education and research should be based on high-quality evidence. Since no decision should be made based on a single study, secondary studies become essential in presenting the evidence. According to our search, over 101 secondary studies have been published in the area of software testing since 1994. With this high number of secondary studies, it is important to conduct a review in this area to provide an overview of the research landscape in this area.
	Objective
	The goal of this study is to systematically map (classify) the secondary studies in software testing. We propose that tertiary studies can serve as summarizing indexes which facilitate finding the most relevant information from secondary studies and thus supporting evidence-based decision making in any given area of software engineering. Our research questions (RQs) investigate: (1) Software-testing-specific areas, (2) Types of RQs investigated, (3) Numbers and Trends, and (4) Citations of the secondary studies.
	Method
	To conduct the tertiary study, we use the systematic-mapping approach. Additionally, we contrast the testing topics to the number of Google hits to address a general popularity of a testing topic and study the most popular papers in terms of citations. We furthermore demonstrate the practicality and usefulness of our results by mapping them to ISTQB foundation syllabus and to SWEBOK to provide implications for practitioners, testing educators, and researchers.
	Results
	After a systematic search and voting process, our study pool included 101 secondary studies in the area of software testing between 1994 and 2015. Among our results are the following: (1) In terms of number of secondary studies, model-based approach is the most popular testing method, web services are the most popular system under test (SUT), while regression testing is the most popular testing phase; (2) The quality of secondary studies, as measured by a criteria set established in the community, is slowly increasing as the years go by; and (3) Analysis of research questions, raised and studied in the pool of secondary studies, showed that there is a lack of ‘causality’ and ‘relationship’ type of research questions, a situation which needs to be improved if we, as a community, want to advance as a scientific field. (4) Among secondary studies, we found that regular surveys receive significantly more citations than SMs (p=0.009) and SLRs (p=0.014).
	Conclusion
	Despite the large number of secondary studies, we found that many important areas of software testing currently lack secondary studies, e.g., test management, role of product risk in testing, human factors in software testing, beta-testing (A/B-testing), exploratory testing, testability, test stopping criteria, and test-environment development. Having secondary studies in those areas is important for satisfying industrial and educational needs in software testing. On the other hand, education material of ISTQB foundation syllabus and SWEBOK could benefit from the inclusion of the latest research topics, namely search-based testing, use of cloud-computing for testing and symbolic execution.}
}

@book{graham2012experiences,
	title={Experiences of test automation: case studies of software test automation},
	author={Graham, Dorothy and Fewster, Mark},
	year={2012},
	publisher={Addison-Wesley Professional}
}

@inproceedings{garousi2018introducing,
	title={Introducing automated GUI testing and observing its benefits: an industrial case study in the context of law-practice management software},
	author={Garousi, Vahid and Yildirim, Erdem},
	booktitle={2018 IEEE International Conference on Software Testing, Verification and Validation Workshops (ICSTW)},
	pages={138--145},
	year={2018},
	organization={IEEE}
}

@article{ramler2018adapting,
	title={Adapting automated test generation to GUI testing of industry applications},
	author={Ramler, Rudolf and Buchgeher, Georg and Klammer, Claus},
	journal={Information and Software Technology},
	volume={93},
	pages={248--263},
	year={2018},
	publisher={Elsevier}
}

@article{lee2018architecture,
	title={Architecture-based software testing},
	author={Lee, Jihyun and Kang, Sungwon and Keum, Changsup},
	journal={International Journal of Software Engineering and Knowledge Engineering},
	volume={28},
	number={01},
	pages={57--77},
	year={2018},
	publisher={World Scientific}
}

@online{mfiles2022Hubshare,
	author = {M-Files},
	title = "M-Files Hubshare",
	year = 2022,
	url = {https://www.m-files.com/solutions/hubshare/},
	urldate = {2022-08-03}
}

@online{hubshare2020,
	author = {Hubshare},
	title = "About Hubshare",
	year = 2020,
	url = {https://web.archive.org/web/20200924221717/https://hubshare.com/en/about-us/},
	urldate = {2022-08-03}
}

@inproceedings{akin2018transitioning,
	author={Akin, Abdurrahman and Sentürk, Serafettin and Garousi, Vahid},
	booktitle={2018 25th Asia-Pacific Software Engineering Conference (APSEC)},
	title={Transitioning from Manual to Automated Software Regression Testing: Experience from the Banking Domain},
	year={2018},
	volume={},
	number={},
	pages={591-597},
	doi={10.1109/APSEC.2018.00074}
}

@article{garousi2016developing,
	author={Garousi, Vahid and Felderer, Michael},
	journal={IEEE Software},
	title={Developing, Verifying, and Maintaining High-Quality Automated Test Scripts},
	year={2016},
	volume={33},
	number={3},
	pages={68-75},
	doi={10.1109/MS.2016.30}
}

@online{mfiles2022tutustu,
	author = {Hubshare},
	title = "Tutustu M-Filesiin",
	year = 2022,
	url = {https://www.m-files.com/fi/tietoja/},
	urldate = {2022-08-07}
}

@misc{dekelbaum2021mfiles,
	title={M-Files Acquires Hubshare to Strengthen External Content Sharing and Collaboration, Deliver Best-in-Class Digital Client Experiences },
	url={https://resources.m-files.com/blog/m-files-acquires-hubshare-to-strengthen-external-content-sharing-and-collaboration-deliver-best-in-class-digital-client-experiences-2},
	journal={M-Files Blog},
	publisher={M-Files},
	author={Dekelbaum, Debra},
	date={2021-04-06}
}

@article{mansikkamäki2017tamperelainen,
	author  = {Mansikkamäki, Enni},
	date    = {2017-03-28},
	title   = {Tamperelainen ohjelmistotalo M-Files laajenee yrityskaupalla Ranskaan – ”Nyt ei yhtään jarrutella”},
	journal = {Aamulehti},
	url     = {https://www.aamulehti.fi/talous/art-2000007376826.html},
	urldate = {2022-08-07}
}

@online{pääomasijoittajat2022mfiles,
	author = {Pääomasijoittajat},
	title = {M-Files - "Päätimme kasvaa vielä rohkeammin"},
	year = 2018,
	url = {https://web.archive.org/web/20180914203646/https://paaomasijoittajat.fi/kasvutarinat/m-files-paatimme-kasvaa-viela-rohkeammin/},
	urldate = {2022-08-07}
}

@inbook{kitchenham1989software,
	title={Software Metrics},
	booktitle={Software Reliability Handbook},
	author={Kitchenham, B},
	journal={London, New York: P. Rook. Elsevier Applied Science},
	year={1989},
	publisher={Elsevier}
}

@inbook{gillies2011software,
	title={Introduction to quality: what is it?},
	booktitle={Software quality: theory and management},
	author={Gillies, Alan},
	year={2011},
	publisher={Lulu. com},
	pages={3-15}
}

@techreport{iso9001,
	type = {Standard},
	key = {ISO 9001:2008(E)},
	month = nov,
	year = {2008},
	title = {Quality management systems - Requirements},
	volume = {2008},
	address = {Geneva, CH},
	institution = {International Organization for Standardization}
}

@inproceedings{petrasch1999definition,
	title={The definition of software quality: a practical approach},
	author={Petrasch, Roland},
	booktitle={Proceedings of the 10th International Symposium on Software Reliability Engineering},
	pages={33--34},
	year={1999}
}

@book{garvin1988managing,
	title={Managing quality: The strategic and competitive edge},
	author={Garvin, David A},
	year={1988},
	publisher={Simon and Schuster}
}

@article{ieee1990glossary,
	author={},
	journal={IEEE Std 610.12-1990},
	title={IEEE Standard Glossary of Software Engineering Terminology},
	year={1990},
	volume={},
	number={},
	pages={1-84},
	doi={10.1109/IEEESTD.1990.101064}
}

@inbook{galin2004software,
	title={What is software quality},
	booktitle={Software quality assurance: from theory to implementation},
	author={Galin, Daniel},
	year={2004},
	pages={14-33},
	publisher={Pearson education}
}

@inbook{tian2005software,
	title={Quality Assurance},
	booktitle={Software quality engineering: testing, quality assurance, and quantifiable improvement},
	author={Tian, Jeff},
	year={2005},
	pages={27-39},
	publisher={John Wiley \& Sons}
}

@article{sharma2014quantitative,
	title={Quantitative analysis of automation and manual testing},
	author={Sharma, RM},
	journal={International journal of engineering and innovative technology},
	volume={4},
	number={1},
	year={2014}
}

@book{ammann2016introduction,
	tittle={Introduction to software testing},
	author={Ammann, Paul and Offutt, Jeff},
	year={2016},
	publisher={Cambridge University Press}
}

@article{gaur2012automated,
	title={How Automated Testing Tools Are Showing Its Impact In The Field Of Software Testing},
	author={Gaur, Deepti and Chhillar, Rajender Singh},
	journal={IJCSMS International Journal of Computer Science \& Management Studies},
	volume={12},
	number={03},
	year={2012},
	publisher={Citeseer}
}

@article{gunasekaran2015survey,
	title={Survey on automation testing tools for mobile applications},
	author={Gunasekaran, S and Bargavi, V},
	journal={International Journal of Advanced Engineering Research and Science},
	volume={2},
	number={11},
	pages={2349--6495},
	year={2015}
}

@inproceedings{berner2005observations,
	author = {Berner, Stefan and Weber, Roland and Keller, Rudolf K.},
	title = {Observations and Lessons Learned from Automated Testing},
	year = {2005},
	isbn = {1581139632},
	publisher = {Association for Computing Machinery},
	address = {New York, NY, USA},
	url = {https://doi.org/10.1145/1062455.1062556},
	doi = {10.1145/1062455.1062556},
	abstract = {This report addresses some of our observations made in a dozen of projects in the area of software testing, and more specifically, in automated testing. It documents, analyzes and consolidates what we consider to be of interest to the community. The major findings can be summarized in a number of lessons learned, covering test strategy, testability, daily integration, and best practices.The report starts with a brief description of five sample projects. Then, we discuss our observations and experiences and illustrate them with the sample projects. The report concludes with a synopsis of these experiences and with suggestions for future test automation endeavors.},
	booktitle = {Proceedings of the 27th International Conference on Software Engineering},
	pages = {571–579},
	numpages = {9},
	keywords = {test management, automated testing, software test},
	location = {St. Louis, MO, USA},
	series = {ICSE '05}
}

@inproceedings{sundaralingam2021analysis,
	author={Sundaralingam, Sakthi and Samantha Rajapaksha, U. U.},
	booktitle={2021 IEEE 16th International Conference on Industrial and Information Systems (ICIIS)},
	title={Analysis on the Risk and the Categorization on Test Automation in Sri Lankan Software Industry},
	year={2021},
	volume={},
	number={},
	pages={312-317},
	doi={10.1109/ICIIS53135.2021.9660637}
}

@inproceedings{timoney2008experiences,
	author={Timoney, Joseph and Brown, Stephen and Ye, Deshi},
	booktitle={2008 The 9th International Conference for Young Computer Scientists},
	title={Experiences in Software Testing Education: Some Observations from an International Cooperation},   year={2008},
	volume={},
	number={},
	pages={2686-2691},
	doi={10.1109/ICYCS.2008.209}
}

@inbook{everett2007software,
	booktitle={Software Testing},
	title={Contents},
	author={Everett, Gerald D and McLeod Jr, Raymond},
	journal={Testing Across the Entire},
	year={2007}
}

@inbook{singh2012software,
	booktitle={Software testing},
	title={Chapter Contents},
	author={Singh, Sanjay Kumar and Singh, Amarjeet},
	year={2012},
	publisher={Vandana Publications}
}

@inproceedings{royce1987managing,
	title={Managing the development of large software systems: concepts and techniques},
	author={Royce, Winston W},
	booktitle={Proceedings of the 9th international conference on Software Engineering},
	pages={328--338},
	year={1987}
}

@book{rubin2012essential,
	title={Essential Scrum: A practical guide to the most popular Agile process},
	author={Rubin, Kenneth S},
	year={2012},
	publisher={Addison-Wesley}
}

@online{beck2001agile,
	author = {Beck, Kent and Beedle, Mike and van Bennekum, Arie and Cockburn, Alistair and Cunningham, Ward and Fowler, Martin and Grenning, James and Highsmith, Jim and Hunt, Andrew and Jeffries, Ron and Marick, Brian and Martin, Robert and Mellor, Steve and Schwaber, Ken and Sutherland, Jeff and Thomas, Dave},
	title = {Manifesto for Agile Software Development},
	year = 2001,
	url = {https://agilemanifesto.org/},
	urldate = {2022-08-27}
}

@book{beck2000extreme,
	title={Extreme programming explained: embrace change},
	author={Beck, Kent},
	year={2000},
	publisher={addison-wesley professional}
}

@book{fewster1999software,
	title={Software test automation},
	author={Fewster, Mark and Graham, Dorothy},
	year={1999},
	publisher={Addison-Wesley Reading}
}

@inproceedings{ieshin2009test,
	author={Ieshin, Alexey and Gerenko, Marina and Dmitriev, Vadim},
	booktitle={2009 5th Central and Eastern European Software Engineering Conference in Russia (CEE-SECR)},
	title={Test automation: Flexible way},
	year={2009},
	volume={},
	number={},
	pages={249-252},
	doi={10.1109/CEE-SECR.2009.5501151}
}

@inproceedings{winkler2018towards,
	author={Winkler, Dietmar and Meixner, Kristof and Biffl, Stefan},
	booktitle={2018 IEEE 23rd International Conference on Emerging Technologies and Factory Automation (ETFA)}, 
	title={Towards Flexible and Automated Testing in Production Systems Engineering Projects},
	year={2018},
	volume={1},
	number={},
	pages={169-176},
	doi={10.1109/ETFA.2018.8502650}
}

@book{pol2002software,
	title={Software testing: A guide to the TMap approach},
	author={Pol, Martin and Teunissen, Ruud and Van Veenendaal, Erik},
	year={2002},
	publisher={Pearson Education}
}

@inbook{dustin2009implementing,
	booktitle={Implementing automated software testing: How to save time and lower costs while raising quality},
	title={Key 5: Implement AST Processes},
	author={Dustin, Elfriede and Garrett, Thom and Gauf, Bernie},
	year={2009},
	publisher={Pearson Education}
}

@inbook{dustin1999automated,
	booktitle={Automated software testing: introduction, management, and performance},
	title={Chapter 4 Automated Testing Introduction Process},
	author={Dustin, Elfriede and Rashka, Jeff and Paul, John},
	year={1999},
	publisher={Addison-Wesley Professional}
}

@article{macario2013test,
	author={Polo, Macario and Reales, Pedro and Piattini, Mario and Ebert, Christof},
	journal={IEEE Software},
	title={Test Automation},
	year={2013},
	volume={30},
	number={1},
	pages={84-89},
	doi={10.1109/MS.2013.15}
}

@inbook{black2007pragmatic,
	title = "3.6 The V Model",
	booktitle = "Pragmatic Software Testing - Becoming an Effective and Efficient Test Professional",
	author = "Black, Rex",
	year = "2007",
	url = "https://app.knovel.com/hotlink/khtml/id:kt011N4WI6/pragmatic-software-testing/the-v-model",
	isbn = "978-0-470-12790-2",
	publisher = {John Wiley \& Sons},
}

@inbook{dick2017Requirements,
	author="Dick, Jeremy
	and Hull, Elizabeth
	and Jackson, Ken",
	title="Introduction",
	bookTitle="Requirements Engineering",
	year="2017",
	publisher="Springer International Publishing",
	address="Cham",
	pages="1--32",
	abstract="This chapter provides an overview of the scope of requirements engineering, including a definition of the key terms.  It describes how requirements engineering fits into the wider discipline of systems engineering, and introduces the central concept of traceability.",
	isbn="978-3-319-61073-3",
	doi="10.1007/978-3-319-61073-3_1",
	url="https://doi.org/10.1007/978-3-319-61073-3_1"
}

@inbook{acharya2014mastering,
	booktitle={Mastering unit testing using Mockito and JUnit},
	title={Chapter 1. JUnit 4 - a Total Recall},
	author={Acharya, Sujoy},
	year={2014},
	publisher={Packt Publishing Ltd}
}

@inbook{huizinga2007automated,
	author={Huizinga, Dorota and Kolawa, Adam},
	booktitle={Automated Defect Prevention: Best Practices in Software Management},
	title={Initial Planning and Infrastructure},
	year={2007},
	volume={},
	number={},
	pages={53-83},
	doi={10.1002/9780470165171.ch3}
}

@inbook{osherove2013art,
	booktitle={The Art of Unit Testing: with examples in C\#},
	title={Part 1. Getting started},
	author={Osherove, Roy},
	year={2013},
	publisher={Simon and Schuster}
}

@book{khorikov2020unit,
	title={Unit Testing Principles, Practices, and Patterns},
	author={Khorikov, Vladimir},
	year={2020},
	publisher={Simon and Schuster}
}

@inbook{simon2014mastering,
	abstract={If you are a developer interested in creating easily maintainable applications that can grow and change with your needs, then this book is for you. Some experience with JavaScript (not necessarily with entire applications written in JavaScript) is required to follow the examples written in the book.},
	author={Simon, Timms},
	isbn={9781783987986},
	publisher={Packt Publishing},
	series={Community Experience Distilled},
	booktitle={Mastering JavaScript Design Patterns},
	title={Chapter 10: Patterns for testing},
	url={https://search.ebscohost.com/login.aspx?direct=true&db=e000xww&AN=910269&site=ehost-live},
	year={2014},
}

@inbook{hunt2003pragmatic,
	booktitle={Pragmatic unit testing in Java with JUnit},
	title={Appendix A Gotchas},
	author={Hunt, Andy and Thomas, Dave},
	year={2003},
	publisher={The Pragmatic Bookshelf}
}

@article{runeson2006survey,
	author={Runeson, P.},
	journal={IEEE Software},
	title={A survey of unit testing practices},
	year={2006},
	volume={23},
	number={4},
	pages={22-29},
	doi={10.1109/MS.2006.91}
}

@inproceedings{brar2015international,
	author={Brar, Hanmeet Kaur and Kaur, Puneet Jai},
	booktitle={2015 2nd International Conference on Computing for Sustainable Global Development (INDIACom)},
	title={Differentiating Integration Testing and unit testing},
	year={2015},
	volume={},
	number={},
	pages={796-798},
	doi={}
}

@article{uma2013software,
	author = {Uma Nath Yadav and Ashutosh Rai and Priyaranjan Verma},
	title = {Software Testing},
	journal={International Journal of Computer Science and Information Technologies},
	year={2013},
	volume={4},
	number={2},
	issn={0975-9646}
}

@article{singh2012approach,
	doi = {10.48550/ARXIV.1207.2718},
	url = {https://arxiv.org/abs/1207.2718},
	author = {Singh, Roopa and Khan, Imran Akhtar},
	keywords = {Software Engineering (cs.SE), FOS: Computer and information sciences, FOS: Computer and information sciences},
	title = {An Approach For Integration Testing In Online Retail Applications},
	publisher = {arXiv},
	year = {2012},
	copyright = {arXiv.org perpetual, non-exclusive license}
}

@inproceedings{vu2018automation,
	title={Automation of Integration Testing of RESTful Hypermedia Systems: A Model-driven Approach.},
	author={Vu, Henry and Fertig, Tobias and Braun, Peter},
	booktitle={WEBIST},
	pages={404--411},
	year={2018}
}

@inproceedings{isha2018automated,
	author={Isha and Sharma, Abhinav and Revathi, M.},
	booktitle={2018 3rd International Conference on Inventive Computation Technologies (ICICT)},
	title={Automated API Testing},
	year={2018},
	volume={},
	number={},
	pages={788-791},
	doi={10.1109/ICICT43934.2018.9034254}
}

@article{bangare2012automated,
	title={Automated API testing approach},
	author={Bangare, Sunil L and Borse, Seema and Bangare, Pallavi S and Nandedkar, Shital},
	journal={International Journal of Engineering Science and Technology},
	volume={4},
	number={2},
	pages={673--676},
	year={2012},
	publisher={Engg Journals Publications}
}

@inproceedings{micco2017state,
	title	= {The State of Continuous Integration Testing @Google},
	author	= {John Micco},
	year	= {2017}
}

@inproceedings{bell2018deflaker,
	author={Bell, Jonathan and Legunsen, Owolabi and Hilton, Michael and Eloussi, Lamyaa and Yung, Tifany and Marinov, Darko},
	booktitle={2018 IEEE/ACM 40th International Conference on Software Engineering (ICSE)},
	title={DeFlaker: Automatically Detecting Flaky Tests},
	year={2018},
	volume={},
	number={},
	pages={433-444},
	doi={10.1145/3180155.3180164}
}

@article{rehman2006software,
	title={Software Component Integration Testing: A Survey},
	author={Rehman, M and Jabeen, Fakhra and Bertolino, Antonia and Polini, Andrea},
	journal={Journal of Software Testing, Verification, and Reliability (STVR)},
	volume={17},
	number={2},
	pages={95--133},
	year={2006}
}

@collection{istqb2018system,
	editor={ISTQB},
	title={ISTQB Glossary},
	year={2018}
}

@inproceedings{borjesson2012automated,
	author={Borjesson, Emil and Feldt, Robert},
	booktitle={2012 IEEE Fifth International Conference on Software Testing, Verification and Validation},
	title={Automated System Testing Using Visual GUI Testing Tools: A Comparative Study in Industry},
	year={2012},
	volume={},
	number={},
	pages={350-359},
	doi={10.1109/ICST.2012.115}
}

@article{banerjee2013graphical,
	title = {Graphical user interface (GUI) testing: Systematic mapping and repository},
	journal = {Information and Software Technology},
	volume = {55},
	number = {10},
	pages = {1679-1694},
	year = {2013},
	issn = {0950-5849},
	doi = {https://doi.org/10.1016/j.infsof.2013.03.004},
	url = {https://www.sciencedirect.com/science/article/pii/S0950584913000669},
	author = {Ishan Banerjee and Bao Nguyen and Vahid Garousi and Atif Memon},
	keywords = {Systematic mapping, GUI application, Testing, Paper repository, Bibliometrics},
	abstract = {Context
	GUI testing is system testing of a software that has a graphical-user interface (GUI) front-end. Because system testing entails that the entire software system, including the user interface, be tested as a whole, during GUI testing, test cases—modeled as sequences of user input events—are developed and executed on the software by exercising the GUI’s widgets (e.g., text boxes and clickable buttons). More than 230 articles have appeared in the area of GUI testing since 1991.
	Objective
	In this paper, we study this existing body of knowledge using a systematic mapping (SM).
	Method
	The SM is conducted using the guidelines proposed by Petersen et al. We pose three sets of research questions. We define selection and exclusion criteria. From the initial pool of 230 articles, published in years 1991–2011, our final pool consisted of 136 articles. We systematically develop a classification scheme and map the selected articles to this scheme.
	Results
	We present two types of results. First, we report the demographics and bibliometrics trends in this domain, including: top-cited articles, active researchers, top venues, and active countries in this research area. Moreover, we derive the trends, for instance, in terms of types of articles, sources of information to derive test cases, types of evaluations used in articles, etc. Our second major result is a publicly-accessible repository that contains all our mapping data. We plan to update this repository on a regular basis, making it a “live” resource for all researchers.
	Conclusion
	Our SM provides an overview of existing GUI testing approaches and helps spot areas in the field that require more attention from the research community. For example, much work is needed to connect academic model-based techniques with commercially available tools. To this end, studies are needed to compare the state-of-the-art in GUI testing in academic techniques and industrial tools.}
}

@article{miller2001acceptance,
	title={Acceptance testing},
	author={Miller, Roy and Collins, Christopher T},
	journal={Proc. XPUniverse},
	volume={238},
	year={2001}
}

@book{binder2000testing,
	title={Testing object-oriented systems: models, patterns, and tools},
	author={Binder, Robert},
	year={2000},
	publisher={Addison-Wesley Professional}
}

@article{alegroth2015visual,
	author={Alégroth, Emil and Feldt, Robert and Ryrholm, Lisa},
	title={Visual GUI testing in practice: challenges, problemsand limitations},
	journal={Empirical Software Engineering},
	year={2015},
	month={06},
	day={01},
	volume={20},
	number={3},
	pages={694-744},
	abstract={In today's software development industry, high-level tests such as Graphical User Interface (GUI) based system and acceptance tests are mostly performed with manual practices that are often costly, tedious and error prone. Test automation has been proposed to solve these problems but most automation techniques approach testing from a lower level of system abstraction. Their suitability for high-level tests has therefore been questioned. High-level test automation techniques such as Record and Replay exist, but studies suggest that these techniques suffer from limitations, e.g. sensitivity to GUI layout or code changes, system implementation dependencies, etc. Visual GUI Testing (VGT) is an emerging technique in industrial practice with perceived higher flexibility and robustness to certain GUI changes than previous high-level (GUI) test automation techniques. The core of VGT is image recognition which is applied to analyze and interact with the bitmap layer of a system's front end. By coupling image recognition with test scripts, VGT tools can emulate end user behavior on almost any GUI-based system, regardless of implementation language, operating system or platform. However, VGT is not without its own challenges, problems and limitations (CPLs) but, like for many other automated test techniques, there is a lack of empirically-based knowledge of these CPLs and how they impact industrial applicability. Crucially, there is also a lack of information on the cost of applying this type of test automation in industry. This manuscript reports an empirical, multi-unit case study performed at two Swedish companies that develop safety-critical software. It studies their transition from manual system test cases into tests automated with VGT. In total, four different test suites that together include more than 300 high-level system test cases were automated for two multi-million lines of code systems. The results show that the transitioned test cases could find defects in the tested systems and that all applicable test cases could be automated. However, during these transition projects a number of hurdles had to be addressed; a total of 58 different CPLs were identified and then categorized into 26 types. We present these CPL types and an analysis of the implications for the transition to and use of VGT in industrial software development practice. In addition, four high-level solutions are presented that were identified during the study, which would address about half of the identified CPLs. Furthermore, collected metrics on cost and return on investment of the VGT transition are reported together with information about the VGT suites' defect finding ability. Nine of the identified defects are reported, 5 of which were unknown to testers with extensive experience from using the manual test suites. The main conclusion from this study is that even though there are many challenges related to the transition and usage of VGT, the technique is still valuable, flexible and considered cost-effective by the industrial practitioners. The presented CPLs also provide decision support in the use and advancement of VGT and potentially other automated testing techniques similar to VGT, e.g. Record and Replay.},
	issn={1573-7616},
	doi={10.1007/s10664-013-9293-5},
	url={https://doi.org/10.1007/s10664-013-9293-5}
}

@inproceedings{paivi2017choosing,
	author = {Raulamo-Jurvanen, Päivi and Mäntylä, Mika and Garousi, Vahid},
	title = {Choosing the Right Test Automation Tool: A Grey Literature Review of Practitioner Sources},
	year = {2017},
	isbn = {9781450348041},
	publisher = {Association for Computing Machinery},
	address = {New York, NY, USA},
	url = {https://doi.org/10.1145/3084226.3084252},
	doi = {10.1145/3084226.3084252},
	pages = {21-30},
	numpages = {10},
	keywords = {test automation tool, Grey literature review, software test automation, tool selection},
	location = {Karlskrona, Sweden},
	series = {EASE\'17}
}

@article{pandit2015agileuat,
	title={AgileUAT: A framework for user acceptance testing based on user stories and acceptance criteria},
	author={Pandit, Pallavi and Tahiliani, Swati},
	journal={International Journal of Computer Applications},
	volume={120},
	number={10},
	year={2015},
	publisher={Citeseer}
}

@article{otaduy2017user,
	title = {User acceptance testing for Agile-developed web-based applications: Empowering customers through wikis and mind maps},
	journal = {Journal of Systems and Software},
	volume = {133},
	pages = {212-229},
	year = {2017},
	issn = {0164-1212},
	doi = {https://doi.org/10.1016/j.jss.2017.01.002},
	url = {https://www.sciencedirect.com/science/article/pii/S016412121730002X},
	author = {I. Otaduy and O. Diaz},
	keywords = {Agile development, User acceptance testing, Test automation},
	abstract = {User Acceptance Testing (UAT) involves validating software in a real setting by the intended audience. The aim is not so much to check the defined requirements but to ensure that the software satisfies the customer’s needs. Agile methodologies put stringent demands on UAT, if only for the frequency at which it needs to be conducted due to the iterative development of small product releases. In this setting, traditional in-person meetings might not scale up well. Complementary ways are needed to reduce the costs of developer-customer collaboration during UAT. This work introduces a wiki-based approach where customers and developers asynchronously collaborate: developers set the UAT scaffolding that will later shepherd customers when testing. To facilitate understanding, mind maps are used to represent UAT sessions. To facilitate engagement, a popular mind map editor, FreeMind, is turned into an editor for FitNesse, the wiki engine in which these ideas are borne out. The approach is evaluated through a case study involving three real customers. First evaluations are promising. Though at different levels of completeness, the three customers were able to complete a UAT. Customers valued asynchronicity, mind map structuredness, and the transparent generation of documentation out of the UAT session.}
}

@misc{fowler2006continuous,
	title={Continuous integration},
	author={Fowler, Martin and Foemmel, Matthew},
	year={2006}
}

@article{meyer2014continuous,
	title={Continuous integration and its tools},
	author={Meyer, Mathias},
	journal={IEEE software},
	volume={31},
	number={3},
	pages={14--16},
	year={2014},
	publisher={IEEE}
}

@Article{laukkanen2018comparison,
	author={Laukkanen, Eero
	and Paasivaara, Maria
	and Itkonen, Juha
	and Lassenius, Casper},
	title={Comparison of release engineering practices in a large mature company and a startup},
	journal={Empirical Software Engineering},
	year={2018},
	month={12},
	day={01},
	volume={23},
	number={6},
	pages={3535-3577},
	abstract={Modern release engineering practices provide multiple benefits for software companies, but organizations have struggled when trying to adopt the most advanced practices, such as continuous delivery. It is not known in which contexts the most advanced practices are applicable and what can be achieved by adopting them. In this study, we discuss the effect of the organizational context on adopted release engineering practices and what outcomes are achieved with the practices. We study two organizational contexts: the startup and the large mature company context. The effect of the product context is mitigated by studying two case organizations with similar products, a rare research opportunity. We performed 18 interviews with various roles in the case organizations. The number of production environments, the number of customers, the control over the production environment, the available resources, the organization size and the distribution of the organization affected the release engineering practices and the ability to release frequently. Having less internal verification and more customer verification enabled fast feedback and customer experimentation in the startup context, but increased the number of production defects. However, having more internal verification in the large mature company context surprisingly did not prevent production defects. The organizational context had a large effect on how achievable modern release engineering practices, such as continuous delivery, were. In the startup context, the lack of resources was the main factor hindering the improvement of release engineering practices, while in the large mature company context, the number of stakeholders and products were the main factors.},
	issn={1573-7616},
	doi={10.1007/s10664-018-9616-7},
	url={https://doi.org/10.1007/s10664-018-9616-7}
}

@inbook{feagin1991case,
	booktitle={A case for the case study},
	title={Introduction},
	author={Feagin, Joe R and Orum, Anthony M and Sjoberg, Gideon},
	year={1991},
	publisher={UNC Press Books}
}

@inbook{flyvbjerg2011case,
	title={Case study},
	author={Flyvbjerg, Bent},
	booktitle={The Sage handbook of qualitative research},
	volume={4},
	pages={301--316},
	year={2011},
	publisher={Thousand Oaks}
}

@article{cassell2005creating,
	title={Creating the interviewer: identity work in the management research process},
	author={Cassell, Catherine},
	journal={Qualitative research},
	volume={5},
	number={2},
	pages={167--179},
	year={2005},
	publisher={Sage Publications Sage CA: Thousand Oaks, CA}
}

@article{alsaawi2014critical,
	title={A critical review of qualitative interviews},
	author={Alsaawi, Ali},
	journal={European Journal of Business and Social Sciences},
	volume={3},
	number={4},
	year={2014}
}

@book{raworth2012conducting,
	title={Conducting semi-structured Interviews},
	author={Raworth, Kate and Sweetman, Caroline and Narayan, Swati and Rowlands, Jo and Hopkins, Adrienne},
	year={2012},
	publisher={Oxfam}
}

@inbook{groves2011survey,
	booktitle={Survey methodology},
	title={Introduction},
	author={Groves, Robert M and Fowler Jr, Floyd J and Couper, Mick P and Lepkowski, James M and Singer, Eleanor and Tourangeau, Roger},
	year={2011},
	publisher={John Wiley \& Sons}
}

@inbook{sapsford2006survey,
	booktitle={Survey research},
	title={Introduction},
	author={Sapsford, Roger},
	year={2006},
	publisher={Sage}
}

@article{firesmith2003specifying,
	title={Specifying good requirements},
	author={Firesmith, Donald},
	journal={Journal of Object Technology},
	volume={2},
	number={4},
	pages={77--87},
	year={2003}
}

@book{olsen2018istqbFoundation,
	title={ISTQB Foundation Level Syllabus},
	author={Olsen, Klaus and Parveen, Tauhida and Black, Rex and Friedenberg, Debra and McKay, Judy and Posthuma, Meile and Schaefer, Hans and Smilgin, Radoslaw and Smith, Mike and Toms, Steve and Ulrich, Stephanie and Walsch, Marie and Zakaria, Eshraka},
	year={2018},
	publisher={International Software Testing Qualifications Board}
}

@inproceedings{karhu2009empirical,
	author={Karhu, Katja and Repo, Tiina and Taipale, Ossi and Smolander, Kari},
	booktitle={2009 International Conference on Software Testing Verification and Validation},
	title={Empirical Observations on Software Testing Automation},
	year={2009},
	volume={},
	number={},
	pages={201-209},
	doi={10.1109/ICST.2009.16}
}

@online{angular2022testing,
	author = {Angular, Google},
	title = {Developer guides, testing},
	year = {2022},
	url = {https://angular.io/guide/testing},
	urldate = {2022-10-12}
}

@online{jest2022testing,
	author = {Facebook Open Source},
	title = {Testing Web Frameworks},
	year = {2022},
	url = {https://jestjs.io/docs/testing-frameworks},
	urldate = {2022-10-12}
}

@online{stateofjs2021,
	author = {Devographics},
	title = {State of JavaScript, Testing},
	year = {2021},
	url = {https://jestjs.io/docs/testing-frameworks},
	urldate = {2022-10-12}
}

@online{cypress2022features,
	author = {Cypress},
	title = {Cypress, features},
	year = {2022},
	url = {https://www.cypress.io/features},
	urldate = {2022-10-13}
}

@online{cypress2022how,
	author = {Cypress},
	title = {Cypress, how it works},
	year = {2022},
	url = {https://www.cypress.io/how-it-works/},
	urldate = {2022-10-13}
}

@inproceedings {oliveira2018blindspots,
	author = {Daniela Seabra Oliveira and Tian Lin and Muhammad Sajidur Rahman and Rad Akefirad and Donovan Ellis and Eliany Perez and Rahul Bobhate and Lois A. DeLong and Justin Cappos and Yuriy Brun},
	title = {{API} Blindspots: Why Experienced Developers Write Vulnerable Code},
	booktitle = {Fourteenth Symposium on Usable Privacy and Security (SOUPS 2018)},
	year = {2018},
	isbn = {978-1-939133-10-6},
	address = {Baltimore, MD},
	pages = {315--328},
	url = {https://www.usenix.org/conference/soups2018/presentation/oliveira},
	publisher = {USENIX Association},
	month = aug,
}

@online{xunit2022home,
	author = {.NET Foundation},
	title = {xUnit home},
	year = {2022},
	url = {https://xunit.net/},
	urldate = {2022-10-21}
}

@online{xunit2022why,
	author = {.NET Foundation},
	title = {Why Did we Build xUnit 1.0},
	year = {2007},
	url = {https://xunit.net/},
	urldate = {2022-10-21}
}

@online{microsoft2022testing,
	author = {Microsoft},
	title = {Testing in .NET},
	year = {2022},
	url = {https://learn.microsoft.com/en-us/dotnet/core/testing/},
	urldate = {2022-10-21}
}

@online{moq2022repository,
	author = {Moq authors},
	title = {Moq repository},
	year = {2022},
	url = {https://github.com/moq/moq4},
	urldate = {2022-10-21}
}

@misc{wake2011aaa,
	title={3A – Arrange, Act, Assert}, url={https://xp123.com/articles/3a-arrange-act-assert/#mobile-header-left-nav},
	journal={XP123},
	author={Wake, Bill},
	year={2011},
	month={4}
}

@misc{scrum2018state,
	title={State of scrum}, url={https://www.scrumalliance.org/ScrumRedesignDEVSite/media/ScrumAllianceMedia/Files%20and%20PDFs/State%20of%20Scrum/2017-SoSR-Final-Version-(Pages).pdf},
	author={Scrum Alliance},
	year={2018}
}

@article{capretz2003personality,
	title = {Personality types in software engineering},
	journal = {International Journal of Human-Computer Studies},
	volume = {58},
	number = {2},
	pages = {207-214},
	year = {2003},
	issn = {1071-5819},
	doi = {https://doi.org/10.1016/S1071-5819(02)00137-4},
	url = {https://www.sciencedirect.com/science/article/pii/S1071581902001374},
	author = {Luiz Fernando Capretz},
	abstract = {Software engineering is forecast to be among the fastest growing employment field in the next decades. The purpose of this investigation is two-fold: Firstly, empirical studies on the personality types of software professionals are reviewed. Secondly, this work provides an up-to-date personality profile of software engineers according to the Myers–Briggs Type Indicator.}
}

@article{marshall1996sampling,
	author = {Marshall, Martin N},
	title = "{Sampling for qualitative research}",
	journal = {Family Practice},
	volume = {13},
	number = {6},
	pages = {522-526},
	year = {1996},
	month = {12},
	abstract = "{The probability sampling techniques used for quantitative studies are rarely appropriate when conducting qualitative research. This article considers and explains the differences between the two approaches and describes three broad categories of naturalistic sampling: convenience, judgement and theoretical models. The principles are illustrated with practical examples from the author's own research.}",
	issn = {0263-2136},
	doi = {10.1093/fampra/13.6.522},
	url = {https://doi.org/10.1093/fampra/13.6.522},
	eprint = {https://academic.oup.com/fampra/article-pdf/13/6/522/6737096/13-6-522.pdf},
}

@article{rising2000scrum,
	author={Rising, L. and Janoff, N.S.},
	journal={IEEE Software},
	title={The Scrum software development process for small teams},
	year={2000},
	volume={17},
	number={4},
	pages={26-32},
	doi={10.1109/52.854065}
}

@article{onoma1998regression,
	author = {Onoma, Akira K. and Tsai, Wei-Tek and Poonawala, Mustafa and Suganuma, Hiroshi},
	title = {Regression Testing in an Industrial Environment},
	year = {1998},
	issue_date = {May 1998},
	publisher = {Association for Computing Machinery},
	address = {New York, NY, USA},
	volume = {41},
	number = {5},
	issn = {0001-0782},
	url = {https://doi.org/10.1145/274946.274960},
	doi = {10.1145/274946.274960},
	journal = {Commun. ACM},
	month = {5},
	pages = {81–86},
	numpages = {6}
}

@book{beck2008practical,
	title={Practical research methods for librarians and information professionals},
	author={Beck, Susan E and Manuel, Kate},
	year={2008},
	publisher={Neal-Schuman Publishers New York}
}

@inproceedings{ricca2021web,
	author="Ricca, Filippo
	and Stocco, Andrea",
	editor="Bure{\v{s}}, Tom{\'a}{\v{s}}
	and Dondi, Riccardo
	and Gamper, Johann
	and Guerrini, Giovanna
	and Jurdzi{\'{n}}ski, Tomasz
	and Pahl, Claus
	and Sikora, Florian
	and Wong, Prudence W.H.",
	title="Web Test Automation: Insights from the Grey Literature",
	booktitle="SOFSEM 2021: Theory and Practice of Computer Science",
	year="2021",
	publisher="Springer International Publishing",
	address="Cham",
	pages="472--485",
	abstract="This paper provides the results of a survey of the grey literature concerning best practices for end-to-end web test automation. We analyzed more than 2,400 sources (e.g., blog posts, white-papers, user manuals, GitHub repositories) looking for guidelines by IT professionals on how to develop and maintain web test code. Ultimately, we filtered 142 relevant documents from which we extracted a taxonomy of guidelines divided into technical tips (i.e., concerning the development, maintenance, and execution of web tests), and business-level tips (i.e, concerning the planning and management of testing teams, design, and process). The paper concludes by distilling the ten most cited best practices for developing good quality automated web tests.",
	isbn="978-3-030-67731-2"
}

@book{jorgensen2013software,
	title={Software testing: a craftsman's approach},
	author={Jorgensen, Paul C},
	year={2013},
	publisher={Auerbach Publications}
}

@book{beck2003test,
	title={Test-driven development: by example},
	author={Beck, Kent},
	year={2003},
	publisher={Addison-Wesley Professional}
}

@article{joshi2015likert,
	title={Likert scale: Explored and explained},
	author={Joshi, Ankur and Kale, Saket and Chandel, Satish and Pal, D Kumar},
	journal={British journal of applied science \& technology},
	volume={7},
	number={4},
	pages={396},
	year={2015},
	publisher={SCIENCEDOMAIN International}
}

@article{edwards2014effects,
	title={The effects of the neutral response option on the extremeness of participant responses},
	author={Edwards, Melinda L and Smith, Brandon C},
	journal={Journal of Undergraduate Scholarship},
	volume={6},
	pages={30},
	year={2014}
}

@article{garland1991mid,
	title={The mid-point on a rating scale: Is it desirable},
	author={Garland, Ron},
	journal={Marketing bulletin},
	volume={2},
	number={1},
	pages={66--70},
	year={1991}
}

@article{burchell1992effect,
	title={The effect of questionnaire length on survey response},
	author={Burchell, Brendan and Marsh, Catherine},
	journal={Quality and quantity},
	volume={26},
	number={3},
	pages={233--244},
	year={1992},
	publisher={Springer}
}

@inbook{hood2007requirements,
	booktitle={Requirements management: The interface between requirements development and all other systems engineering processes},
	title={Why Requirements Management and Engineering},
	author={Hood, Colin and Wiedemann, Simon and Fichtinger, Stefan and Pautz, Urte},
	year={2007},
	publisher={Springer Science \& Business Media}
}

@online{brown2022c4,
	author = {Brown, Simon},
	title = {The C4 model for visualising software architecture},
	year = {2022},
	url = {https://c4model.com/},
	urldate = {2022-12-08}
}

 @misc{stefanovskyi2019unit,
 	title={Unit Test Naming Conventions},
 	url={https://medium.com/@stefanovskyi/unit-test-naming-conventions-dd9208eadbea},
 	journal={Medium},
 	author={Stefanovskyi, Oleksandr},
 	year={2019},
 	month={1}
 }