{"id":"https://openalex.org/W3118288752","doi":"https://doi.org/10.1109/thms.2022.3149173","title":"A Review of Evaluation Practices of Gesture Generation in Embodied Conversational Agents","display_name":"A Review of Evaluation Practices of Gesture Generation in Embodied Conversational Agents","publication_year":2022,"publication_date":"2022-02-25","ids":{"openalex":"https://openalex.org/W3118288752","doi":"https://doi.org/10.1109/thms.2022.3149173","mag":"3118288752"},"language":"en","primary_location":{"is_oa":false,"landing_page_url":"https://doi.org/10.1109/thms.2022.3149173","pdf_url":null,"source":{"id":"https://openalex.org/S2476799526","display_name":"IEEE Transactions on Human-Machine Systems","issn_l":"2168-2291","issn":["2168-2291","2168-2305"],"is_oa":false,"is_in_doaj":false,"is_indexed_in_scopus":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false},"type":"review","type_crossref":"journal-article","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"http://arxiv.org/pdf/2101.03769","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5053778039","display_name":"Pieter Wolfert","orcid":"https://orcid.org/0000-0002-7420-7181"},"institutions":[{"id":"https://openalex.org/I32597200","display_name":"Ghent University","ror":"https://ror.org/00cv9y106","country_code":"BE","type":"funder","lineage":["https://openalex.org/I32597200"]}],"countries":["BE"],"is_corresponding":false,"raw_author_name":"Pieter Wolfert","raw_affiliation_strings":["IDLab, Ghent University, Gent, Belgium"],"affiliations":[{"raw_affiliation_string":"IDLab, Ghent University, Gent, Belgium","institution_ids":["https://openalex.org/I32597200"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038581145","display_name":"Nicole Robinson","orcid":"https://orcid.org/0000-0002-7144-3082"},"institutions":[{"id":"https://openalex.org/I56590836","display_name":"Monash University","ror":"https://ror.org/02bfwt286","country_code":"AU","type":"funder","lineage":["https://openalex.org/I56590836"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Nicole Robinson","raw_affiliation_strings":["Department of Electrical and Computer Systems Engineering, Turner Institute for Brain and Mental Health, Faculty of Engineering, Faculty of Medicine, Nursing and Health Sciences, Monash University, Clayton, Victoria, Australia"],"affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Systems Engineering, Turner Institute for Brain and Mental Health, Faculty of Engineering, Faculty of Medicine, Nursing and Health Sciences, Monash University, Clayton, Victoria, Australia","institution_ids":["https://openalex.org/I56590836"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5035627933","display_name":"Tony Belpaeme","orcid":"https://orcid.org/0000-0001-5207-7745"},"institutions":[{"id":"https://openalex.org/I32597200","display_name":"Ghent University","ror":"https://ror.org/00cv9y106","country_code":"BE","type":"funder","lineage":["https://openalex.org/I32597200"]}],"countries":["BE"],"is_corresponding":false,"raw_author_name":"Tony Belpaeme","raw_affiliation_strings":["IDLab, Ghent University, Gent, Belgium"],"affiliations":[{"raw_affiliation_string":"IDLab, Ghent University, Gent, Belgium","institution_ids":["https://openalex.org/I32597200"]}]}],"institution_assertions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.541,"has_fulltext":true,"fulltext_origin":"pdf","cited_by_count":59,"citation_normalized_percentile":{"value":0.606127,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":98,"max":99},"biblio":{"volume":"52","issue":"3","first_page":"379","last_page":"389"},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T10709","display_name":"Social Robot Interaction and HRI","score":0.9998,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T10709","display_name":"Social Robot Interaction and HRI","score":0.9998,"subfield":{"id":"https://openalex.org/subfields/3207","display_name":"Social Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12031","display_name":"Speech and dialogue systems","score":0.9964,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12128","display_name":"AI in Service Interactions","score":0.9959,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[],"concepts":[{"id":"https://openalex.org/C100609095","wikidata":"https://www.wikidata.org/wiki/Q1335050","display_name":"Embodied cognition","level":2,"score":0.90513116},{"id":"https://openalex.org/C207347870","wikidata":"https://www.wikidata.org/wiki/Q371174","display_name":"Gesture","level":2,"score":0.77351487},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.48567832},{"id":"https://openalex.org/C107457646","wikidata":"https://www.wikidata.org/wiki/Q207434","display_name":"Human\u2013computer interaction","level":1,"score":0.38803995},{"id":"https://openalex.org/C46312422","wikidata":"https://www.wikidata.org/wiki/Q11024","display_name":"Communication","level":1,"score":0.37468928},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.3496713},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.3276832},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.1898596},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.077319086}],"mesh":[],"locations_count":5,"locations":[{"is_oa":false,"landing_page_url":"https://doi.org/10.1109/thms.2022.3149173","pdf_url":null,"source":{"id":"https://openalex.org/S2476799526","display_name":"IEEE Transactions on Human-Machine Systems","issn_l":"2168-2291","issn":["2168-2291","2168-2305"],"is_oa":false,"is_in_doaj":false,"is_indexed_in_scopus":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false},{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2101.03769","pdf_url":"http://arxiv.org/pdf/2101.03769","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false},{"is_oa":true,"landing_page_url":"http://hdl.handle.net/1854/LU-8743014","pdf_url":"https://biblio.ugent.be/publication/8743014/file/8743015.pdf","source":{"id":"https://openalex.org/S4306400478","display_name":"Ghent University Academic Bibliography (Ghent University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I32597200","host_organization_name":"Ghent University","host_organization_lineage":["https://openalex.org/I32597200"],"host_organization_lineage_names":["Ghent University"],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"publishedVersion","is_accepted":true,"is_published":true},{"is_oa":true,"landing_page_url":"https://biblio.ugent.be/publication/8743014/file/8743015","pdf_url":"https://biblio.ugent.be/publication/8743014/file/8743015","source":{"id":"https://openalex.org/S4306400478","display_name":"Ghent University Academic Bibliography (Ghent University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I32597200","host_organization_name":"Ghent University","host_organization_lineage":["https://openalex.org/I32597200"],"host_organization_lineage_names":["Ghent University"],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"publishedVersion","is_accepted":true,"is_published":true},{"is_oa":false,"landing_page_url":"https://api.datacite.org/dois/10.48550/arxiv.2101.03769","pdf_url":null,"source":{"id":"https://openalex.org/S4393179698","display_name":"DataCite API","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I4210145204","host_organization_name":"DataCite","host_organization_lineage":["https://openalex.org/I4210145204"],"host_organization_lineage_names":["DataCite"],"type":"metadata"},"license":null,"license_id":null,"version":null}],"best_oa_location":{"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2101.03769","pdf_url":"http://arxiv.org/pdf/2101.03769","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_indexed_in_scopus":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false},"sustainable_development_goals":[{"display_name":"Quality education","score":0.61,"id":"https://metadata.un.org/sdg/4"}],"grants":[{"funder":"https://openalex.org/F4320321730","funder_display_name":"Fonds Wetenschappelijk Onderzoek","award_id":"1S95020N"},{"funder":"https://openalex.org/F4320327336","funder_display_name":"Vlaamse regering","award_id":null}],"datasets":[],"versions":["https://openalex.org/W3118288752"],"referenced_works_count":99,"referenced_works":["https://openalex.org/W1534876773","https://openalex.org/W153909638","https://openalex.org/W1544484417","https://openalex.org/W1548076732","https://openalex.org/W1563542286","https://openalex.org/W164929510","https://openalex.org/W1697749196","https://openalex.org/W1969681536","https://openalex.org/W1973418305","https://openalex.org/W1979531823","https://openalex.org/W2007872832","https://openalex.org/W2008208299","https://openalex.org/W2012844124","https://openalex.org/W2017354274","https://openalex.org/W2024649868","https://openalex.org/W2032233641","https://openalex.org/W2032568497","https://openalex.org/W2035046981","https://openalex.org/W2037926088","https://openalex.org/W2039376592","https://openalex.org/W2045053352","https://openalex.org/W2070269115","https://openalex.org/W2071582555","https://openalex.org/W2077223959","https://openalex.org/W2077257649","https://openalex.org/W2078082557","https://openalex.org/W2116966274","https://openalex.org/W2118163921","https://openalex.org/W2118605853","https://openalex.org/W2122376170","https://openalex.org/W2126558800","https://openalex.org/W2128848653","https://openalex.org/W2131734538","https://openalex.org/W2132891289","https://openalex.org/W2135431835","https://openalex.org/W2141125555","https://openalex.org/W2141708418","https://openalex.org/W2161460193","https://openalex.org/W2161676006","https://openalex.org/W2185776260","https://openalex.org/W2296270309","https://openalex.org/W2296371640","https://openalex.org/W2403268709","https://openalex.org/W2468996569","https://openalex.org/W2495266147","https://openalex.org/W2549029155","https://openalex.org/W2561531858","https://openalex.org/W2570203519","https://openalex.org/W2766709201","https://openalex.org/W2803227857","https://openalex.org/W2871950322","https://openalex.org/W2884374054","https://openalex.org/W2899929018","https://openalex.org/W2901468574","https://openalex.org/W2901872500","https://openalex.org/W2904868363","https://openalex.org/W2921888053","https://openalex.org/W2922127468","https://openalex.org/W2922298118","https://openalex.org/W2967443589","https://openalex.org/W2971774262","https://openalex.org/W2972478411","https://openalex.org/W2981802563","https://openalex.org/W2986302981","https://openalex.org/W2988952730","https://openalex.org/W2996457820","https://openalex.org/W2999042931","https://openalex.org/W3001517505","https://openalex.org/W3002310794","https://openalex.org/W3003307838","https://openalex.org/W3009042479","https://openalex.org/W3014235025","https://openalex.org/W3025006235","https://openalex.org/W3083173864","https://openalex.org/W3093647117","https://openalex.org/W3095030303","https://openalex.org/W3099073275","https://openalex.org/W3101175133","https://openalex.org/W3121518240","https://openalex.org/W3122519107","https://openalex.org/W3125775899","https://openalex.org/W3143073753","https://openalex.org/W3153551559","https://openalex.org/W3160751818","https://openalex.org/W3198642229","https://openalex.org/W3205214582","https://openalex.org/W3212693919","https://openalex.org/W3676479","https://openalex.org/W4211164456","https://openalex.org/W4214544731","https://openalex.org/W4214908293","https://openalex.org/W4230193733","https://openalex.org/W4230312136","https://openalex.org/W4248915016","https://openalex.org/W4288079519","https://openalex.org/W4299390018","https://openalex.org/W4300879629","https://openalex.org/W4307080166","https://openalex.org/W4377237746"],"related_works":["https://openalex.org/W4254718650","https://openalex.org/W4244451296","https://openalex.org/W2946643217","https://openalex.org/W2745478210","https://openalex.org/W2473026536","https://openalex.org/W2142421361","https://openalex.org/W2042499210","https://openalex.org/W2036100805","https://openalex.org/W2019233767","https://openalex.org/W14374595"],"abstract_inverted_index":{"Embodied":[0],"conversational":[1],"agents":[2],"(ECA)":[3],"are":[4,40],"often":[5],"designed":[6],"to":[7,11,145,196,214,227],"produce":[8],"nonverbal":[9,22],"behavior":[10,23],"complement":[12],"or":[13],"enhance":[14],"their":[15],"verbal":[16,43],"communication.":[17,44],"One":[18],"such":[19],"form":[20,160],"of":[21,68,87,161,221],"is":[24],"co-speech":[25,102,132,181,229],"gesturing,":[26],"which":[27,90],"involves":[28],"movements":[29],"that":[30,39,121,130,141,171,210],"the":[31,63,69,73,172,219],"agent":[32],"makes":[33],"with":[34,42,62,125],"its":[35],"arms":[36],"and":[37,59,110,156,177,184,193],"hands":[38],"paired":[41],"Co-speech":[45],"gestures":[46],"for":[47,106,180,187,218],"ECAs":[48],"can":[49,211],"be":[50,212],"created":[51],"using":[52],"different":[53],"generation":[54,82,104],"methods,":[55],"divided":[56],"into":[57],"rule-based":[58],"data-driven":[60],"processes,":[61],"latter":[64],"gaining":[65],"traction":[66],"because":[67],"increasing":[70],"interest":[71],"from":[72],"applied":[74],"machine":[75],"learning":[76],"community.":[77],"However,":[78],"reports":[79],"on":[80,101,158],"gesture":[81,103,182,230],"methods":[83,105],"use":[84,142,152],"a":[85,98,126,153,159,166,208],"variety":[86],"evaluation":[88,115,220],"measures,":[89],"hinders":[91],"comparison.":[92],"To":[93],"address":[94],"this,":[95],"we":[96,205],"present":[97],"systematic":[99,167],"review":[100,118],"iconic,":[107],"metaphoric,":[108],"deictic,":[109],"beat":[111],"gestures,":[112],"including":[113,190],"reported":[114],"methods.":[116],"We":[117,148,169],"22":[119],"studies":[120,140,151],"have":[122],"an":[123],"ECA":[124],"human-like":[127],"upper":[128],"body":[129],"uses":[131],"gesturing":[133],"in":[134],"social":[135],"human-agent":[136],"interaction.":[137],"This":[138],"includes":[139],"human":[143],"participants":[144],"evaluate":[146,228],"performance.":[147],"found":[149],"most":[150],"within-subject":[154],"design":[155],"rely":[157],"subjective":[162],"evaluation,":[163,183,189],"but":[164],"without":[165],"approach.":[168],"argue":[170],"field":[173],"requires":[174],"more":[175],"rigorous":[176],"uniform":[178],"tools":[179],"formulate":[185],"recommendations":[186],"empirical":[188],"standardized":[191],"phrases":[192],"example":[194],"scenarios":[195],"help":[197],"systematically":[198],"test":[199],"generative":[200,222],"models":[201],"across":[202],"studies.":[203],"Furthermore,":[204],"also":[206],"propose":[207],"checklist":[209],"used":[213],"report":[215],"relevant":[216],"information":[217],"models,":[223],"as":[224,226],"well":[225],"use.":[231]},"abstract_inverted_index_v3":null,"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W3118288752","counts_by_year":[{"year":2024,"cited_by_count":20},{"year":2023,"cited_by_count":19},{"year":2022,"cited_by_count":16},{"year":2021,"cited_by_count":4}],"updated_date":"2025-03-17T00:14:44.813333","created_date":"2021-01-18"}