{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2024,9,8]],"date-time":"2024-09-08T09:44:00Z","timestamp":1725788640496},"publisher-location":"New York, NY, USA","reference-count":25,"publisher":"ACM","content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2024,7,10]]},"DOI":"10.1145\/3664646.3664766","type":"proceedings-article","created":{"date-parts":[[2024,7,10]],"date-time":"2024-07-10T19:39:56Z","timestamp":1720640396000},"page":"74-78","update-policy":"http:\/\/dx.doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":0,"title":["An AI System Evaluation Framework for Advancing AI Safety: Terminology, Taxonomy, Lifecycle Mapping"],"prefix":"10.1145","author":[{"ORCID":"http:\/\/orcid.org\/0009-0003-7385-4023","authenticated-orcid":false,"given":"Boming","family":"Xia","sequence":"first","affiliation":[{"name":"CSIRO's Data61, Sydney, Australia \/ University of New South Wales, Sydney, Australia"}]},{"ORCID":"http:\/\/orcid.org\/0000-0002-9466-1672","authenticated-orcid":false,"given":"Qinghua","family":"Lu","sequence":"additional","affiliation":[{"name":"CSIRO's Data61, Sydney, Australia \/ University of New South Wales, Sydney, Australia"}]},{"ORCID":"http:\/\/orcid.org\/0000-0001-5839-3765","authenticated-orcid":false,"given":"Liming","family":"Zhu","sequence":"additional","affiliation":[{"name":"CSIRO's Data61, Sydney, Australia \/ University of New South Wales, Sydney, Australia"}]},{"ORCID":"http:\/\/orcid.org\/0000-0001-7663-1421","authenticated-orcid":false,"given":"Zhenchang","family":"Xing","sequence":"additional","affiliation":[{"name":"CSIRO's Data61, Canberra, Australia \/ Australian National University, Canberra, Australia"}]}],"member":"320","published-online":{"date-parts":[[2024,7,10]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"2016. Regulation (EU) 2016\/679 of the European Parliament and of the Council of 27 April 2016 on the Protection of Natural Persons with regard to the Processing of Personal Data and on the Free Movement of Such Data and Repealing Directive 95\/46\/EC (General Data Protection Regulation)."},{"key":"e_1_3_2_1_2_1","unstructured":"2022. ISO\/IEC 22989:2022 Information technology \u2013 Artificial intelligence \u2013 Artificial intelligence concepts and terminology."},{"key":"e_1_3_2_1_3_1","unstructured":"2023. The Bletchley Declaration by Countries Attending the AI Safety Summit."},{"key":"e_1_3_2_1_4_1","unstructured":"AI Safety Institute. 2024. AI Safety Institute Approach to Evaluations."},{"key":"e_1_3_2_1_5_1","volume-title":"Ya-Qin Zhang, Lan Xue, Shai Shalev-Shwartz, and Gillian Hadfield.","author":"Bengio Yoshua","year":"2023","unstructured":"Yoshua Bengio, Geoffrey Hinton, Andrew Yao, Dawn Song, Pieter Abbeel, Yuval Noah Harari, Ya-Qin Zhang, Lan Xue, Shai Shalev-Shwartz, and Gillian Hadfield. 2023. Managing ai risks in an era of rapid progress. arXiv preprint arXiv:2310.17688."},{"key":"e_1_3_2_1_6_1","volume-title":"Fairness Testing: A Comprehensive Survey and Analysis of Trends. ACM Transactions on Software Engineering and Methodology.","author":"Chen Zhenpeng","year":"2023","unstructured":"Zhenpeng Chen, Jie M Zhang, Max Hort, Mark Harman, and Federica Sarro. 2023. Fairness Testing: A Comprehensive Survey and Analysis of Trends. ACM Transactions on Software Engineering and Methodology."},{"key":"e_1_3_2_1_7_1","unstructured":"Jane Huang Kirk Li and Daniel Yehdego. 2024. Evaluating LLM Systems: Metrics Challenges and Best Practices."},{"volume-title":"Guide to the Software Engineering Body of Knowledge (SWEBOK): Version 3.0","author":"IEEE Computer Society","key":"e_1_3_2_1_8_1","unstructured":"IEEE Computer Society. 2014. Guide to the Software Engineering Body of Knowledge (SWEBOK): Version 3.0. IEEE Computer Society."},{"key":"e_1_3_2_1_9_1","volume-title":"Responsible AI: Best Practices for Creating Trustworthy AI Systems","author":"Lu Qinghua","year":"2023","unstructured":"Qinghua Lu, Liming Zhu, Jon Whittle, and Xiwei Xu. 2023. Responsible AI: Best Practices for Creating Trustworthy AI Systems. Addison-Wesley Professional."},{"key":"e_1_3_2_1_10_1","unstructured":"Qinghua Lu Liming Zhu Xiwei Xu Jon Whittle Didar Zowghi and Aurelie Jacquet. 2023. Responsible ai pattern catalogue: A collection of best practices for ai governance and engineering. Comput. Surveys."},{"key":"e_1_3_2_1_11_1","volume-title":"Selfcheckgpt: Zero-resource black-box hallucination detection for generative large language models. arXiv preprint arXiv:2303.08896.","author":"Manakul Potsawee","year":"2023","unstructured":"Potsawee Manakul, Adian Liusie, and Mark JF Gales. 2023. Selfcheckgpt: Zero-resource black-box hallucination detection for generative large language models. arXiv preprint arXiv:2303.08896."},{"key":"e_1_3_2_1_12_1","unstructured":"US National Institute of Standards and Technology (NIST). 2023. AI Risk Management Framework (AI RMF 1.0)."},{"key":"e_1_3_2_1_13_1","volume-title":"Proceedings of the 40th annual meeting of the Association for Computational Linguistics. 311\u2013318","author":"Papineni Kishore","year":"2002","unstructured":"Kishore Papineni, Salim Roukos, Todd Ward, and Wei-Jing Zhu. 2002. Bleu: a method for automatic evaluation of machine translation. In Proceedings of the 40th annual meeting of the Association for Computational Linguistics. 311\u2013318."},{"key":"e_1_3_2_1_14_1","volume-title":"Dan Braun, J\u00e9r\u00e9my Scheurer, Mikita Balesni, Lucius Bushnaq, Charlotte Stix, and Marius Hobbhahn.","author":"Sharkey Lee","year":"2024","unstructured":"Lee Sharkey, Cl\u00edodhna N\u00ed Ghuidhir, Dan Braun, J\u00e9r\u00e9my Scheurer, Mikita Balesni, Lucius Bushnaq, Charlotte Stix, and Marius Hobbhahn. 2024. A Causal Framework for AI Regulation and Auditing."},{"key":"e_1_3_2_1_15_1","volume-title":"Trustllm: Trustworthiness in large language models. arXiv preprint arXiv:2401.05561.","author":"Sun Lichao","year":"2024","unstructured":"Lichao Sun, Yue Huang, Haoran Wang, Siyuan Wu, Qihui Zhang, Chujie Gao, Yixin Huang, Wenhan Lyu, Yixuan Zhang, and Xiner Li. 2024. Trustllm: Trustworthiness in large language models. arXiv preprint arXiv:2401.05561."},{"key":"e_1_3_2_1_16_1","unstructured":"The White House. 2023. Executive Order on the Safe Secure and Trustworthy Development and Use of Artificial Intelligence."},{"key":"e_1_3_2_1_17_1","unstructured":"Bertie Vidgen Adarsh Agrawal Ahmed M Ahmed Victor Akinwande Namir Al-Nuaimi Najla Alfaraj Elie Alhajjar Lora Aroyo Trupti Bavalatti and Borhane Blili-Hamelin. 2024. Introducing v0. 5 of the ai safety benchmark from mlcommons. arXiv preprint arXiv:2404.12241."},{"key":"e_1_3_2_1_18_1","volume-title":"GLUE: A multi-task benchmark and analysis platform for natural language understanding. arXiv preprint arXiv:1804.07461.","author":"Wang Alex","year":"2018","unstructured":"Alex Wang, Amanpreet Singh, Julian Michael, Felix Hill, Omer Levy, and Samuel R Bowman. 2018. GLUE: A multi-task benchmark and analysis platform for natural language understanding. arXiv preprint arXiv:1804.07461."},{"key":"e_1_3_2_1_19_1","volume-title":"Decodingtrust: A comprehensive assessment of trustworthiness in gpt models. arXiv preprint arXiv:2306.11698.","author":"Wang Boxin","year":"2023","unstructured":"Boxin Wang, Weixin Chen, Hengzhi Pei, Chulin Xie, Mintong Kang, Chenhui Zhang, Chejian Xu, Zidi Xiong, Ritik Dutta, and Rylan Schaeffer. 2023. Decodingtrust: A comprehensive assessment of trustworthiness in gpt models. arXiv preprint arXiv:2306.11698."},{"key":"e_1_3_2_1_20_1","volume-title":"Juan Mateos-Garcia, Stevie Bergman, Jackie Kay, Conor Griffin, and Ben Bariach.","author":"Weidinger Laura","year":"2023","unstructured":"Laura Weidinger, Maribeth Rauh, Nahema Marchal, Arianna Manzini, Lisa Anne Hendricks, Juan Mateos-Garcia, Stevie Bergman, Jackie Kay, Conor Griffin, and Ben Bariach. 2023. Sociotechnical safety evaluation of generative ai systems. arXiv preprint arXiv:2310.11986."},{"key":"e_1_3_2_1_21_1","volume-title":"3rd International Conference on AI Engineering\u2013Software Engineering for AI (CAIN \u201924)","author":"Xia Boming","year":"2024","unstructured":"Boming Xia, Qinghua Lu, Liming Zhu, Sung Une Lee, Yue Liu, and Zhenchang Xing. 2024. Towards a Responsible AI Metrics Catalogue: A Collection of Metrics for AI Accountability. In 3rd International Conference on AI Engineering\u2013Software Engineering for AI (CAIN \u201924)."},{"key":"e_1_3_2_1_22_1","volume-title":"Heather Miller, Chris Potts, James Zou, Michael Carbin, Jonathan Frankle, Naveen Rao, and Ali Ghodsi.","author":"Zaharia Matei","year":"2024","unstructured":"Matei Zaharia, Omar Khattab, Lingjiao Chen, Jared Quincy Davis, Heather Miller, Chris Potts, James Zou, Michael Carbin, Jonathan Frankle, Naveen Rao, and Ali Ghodsi. 2024. The Shift from Models to Compound AI Systems."},{"key":"e_1_3_2_1_23_1","unstructured":"Dawen Zhang Boming Xia Yue Liu Xiwei Xu Thong Hoang Zhenchang Xing Mark Staples Qinghua Lu and Liming Zhu. 2023. Navigating privacy and copyright challenges across the data lifecycle of generative ai. arXiv preprint arXiv:2311.18252."},{"key":"e_1_3_2_1_24_1","unstructured":"Zhexin Zhang Yida Lu Jingyuan Ma Di Zhang Rui Li Pei Ke Hao Sun Lei Sha Zhifang Sui and Hongning Wang. 2024. ShieldLM: Empowering LLMs as Aligned Customizable and Explainable Safety Detectors. arXiv preprint arXiv:2402.16444."},{"key":"e_1_3_2_1_25_1","volume-title":"Xu Chen, Yankai Lin, Ji-Rong Wen, and Jiawei Han.","author":"Zhou Kun","year":"2023","unstructured":"Kun Zhou, Yutao Zhu, Zhipeng Chen, Wentong Chen, Wayne Xin Zhao, Xu Chen, Yankai Lin, Ji-Rong Wen, and Jiawei Han. 2023. Don\u2019t Make Your LLM an Evaluation Benchmark Cheater. arXiv preprint arXiv:2311.01964."}],"event":{"name":"AIware '24: 1st ACM International Conference on AI-Powered Software","sponsor":["SIGSOFT ACM Special Interest Group on Software Engineering"],"location":"Porto de Galinhas Brazil","acronym":"AIware '24"},"container-title":["Proceedings of the 1st ACM International Conference on AI-Powered Software"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3664646.3664766","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,7,10]],"date-time":"2024-07-10T19:43:36Z","timestamp":1720640616000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3664646.3664766"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,7,10]]},"references-count":25,"alternative-id":["10.1145\/3664646.3664766","10.1145\/3664646"],"URL":"https:\/\/doi.org\/10.1145\/3664646.3664766","relation":{},"subject":[],"published":{"date-parts":[[2024,7,10]]},"assertion":[{"value":"2024-07-10","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}