{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2024,4,24]],"date-time":"2024-04-24T17:25:55Z","timestamp":1713979555299},"reference-count":37,"publisher":"Springer Science and Business Media LLC","issue":"1","license":[{"start":{"date-parts":[[2024,3,12]],"date-time":"2024-03-12T00:00:00Z","timestamp":1710201600000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"},{"start":{"date-parts":[[2024,3,12]],"date-time":"2024-03-12T00:00:00Z","timestamp":1710201600000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"}],"funder":[{"name":"Zhejiang Provincial Natural Science Foundation of China","award":["LY21F020017"]},{"name":"the ERC IMI","award":["101005122"]},{"DOI":"10.13039\/100010686","name":"H2020 European Institute of Innovation and Technology","doi-asserted-by":"publisher","award":["952172"],"id":[{"id":"10.13039\/100010686","id-type":"DOI","asserted-by":"publisher"}]},{"name":"the UKRI Future Leaders Fellowship","award":["MR\/V023799\/1"]}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["J Cheminform"],"abstract":"Abstract<\/jats:title>Chemical structure segmentation constitutes a pivotal task in cheminformatics, involving the extraction and abstraction of structural information of chemical compounds from text-based sources, including patents and scientific articles. This study introduces a deep learning approach to chemical structure segmentation, employing a Vision Transformer (ViT) to discern the structural patterns of chemical compounds from their graphical representations. The Chemistry-Segment Anything Model (ChemSAM) achieves state-of-the-art results on publicly available benchmark datasets and real-world tasks, underscoring its effectiveness in accurately segmenting chemical structures from text-based sources. Moreover, this deep learning-based approach obviates the need for handcrafted features and demonstrates robustness against variations in image quality and style. During the detection phase, a ViT-based encoder-decoder model is used to identify and locate chemical structure depictions on the input page. This model generates masks to ascertain whether each pixel belongs to a chemical structure, thereby offering a pixel-level classification and indicating the presence or absence of chemical structures at each position. Subsequently, the generated masks are clustered based on their connectivity, and each mask cluster is updated to encapsulate a single structure in the post-processing workflow. This two-step process facilitates the effective automatic extraction of chemical structure depictions from documents. By utilizing the deep learning approach described herein, it is demonstrated that effective performance on low-resolution and densely arranged molecular structural layouts in journal articles and patents is achievable.<\/jats:p>","DOI":"10.1186\/s13321-024-00823-2","type":"journal-article","created":{"date-parts":[[2024,3,13]],"date-time":"2024-03-13T00:02:45Z","timestamp":1710288165000},"update-policy":"http:\/\/dx.doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Automated molecular structure segmentation from documents using ChemSAM"],"prefix":"10.1186","volume":"16","author":[{"given":"Bowen","family":"Tang","sequence":"first","affiliation":[]},{"given":"Zhangming","family":"Niu","sequence":"additional","affiliation":[]},{"given":"Xiaofeng","family":"Wang","sequence":"additional","affiliation":[]},{"given":"Junjie","family":"Huang","sequence":"additional","affiliation":[]},{"given":"Chao","family":"Ma","sequence":"additional","affiliation":[]},{"given":"Jing","family":"Peng","sequence":"additional","affiliation":[]},{"given":"Yinghui","family":"Jiang","sequence":"additional","affiliation":[]},{"given":"Ruiquan","family":"Ge","sequence":"additional","affiliation":[]},{"given":"Hongyu","family":"Hu","sequence":"additional","affiliation":[]},{"given":"Luhao","family":"Lin","sequence":"additional","affiliation":[]},{"given":"Guang","family":"Yang","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2024,3,12]]},"reference":[{"key":"823_CR1","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1186\/1752-153X-3-4","volume":"3","author":"J Park","year":"2009","unstructured":"Park J, Rosania GR, Shedden KA, Nguyen M, Lyu N, Saitou K (2009) Automated extraction of chemical structure information from digital raster images. Chem Cent J 3:1\u201316","journal-title":"Chem Cent J"},{"issue":"3","key":"823_CR2","doi-asserted-by":"publisher","first-page":"338","DOI":"10.1021\/ci00013a010","volume":"33","author":"P Ibison","year":"1993","unstructured":"Ibison P, Jacquot M, Kam F, Neville A, Simpson RW, Tonnelier C, Venczel T, Johnson AP (1993) Chemical literature data extraction: the CLiDE project. J Chem Inf Comput Sci 33(3):338\u2013344","journal-title":"J Chem Inf Comput Sci"},{"key":"823_CR3","doi-asserted-by":"crossref","unstructured":"Sadawi NM, Sexton AP, Sorge V (2012) Chemical structure recognition: a rule-based approach. In: Document recognition and retrieval XIX. SPIE, pp 101\u2013109","DOI":"10.1117\/12.912185"},{"issue":"4","key":"823_CR4","doi-asserted-by":"publisher","first-page":"780","DOI":"10.1021\/ci800449t","volume":"49","author":"AT Valko","year":"2009","unstructured":"Valko AT, Johnson AP (2009) CLiDE Pro: the latest generation of CLiDE, a tool for optical chemical structure recognition. J Chem Inf Model 49(4):780\u2013787","journal-title":"J Chem Inf Model"},{"issue":"4","key":"823_CR5","doi-asserted-by":"publisher","first-page":"373","DOI":"10.1021\/ci00008a018","volume":"32","author":"JR McDaniel","year":"1992","unstructured":"McDaniel JR, Balmuth JR (1992) Kekule: OCR-optical chemical (structure) recognition. J Chem Inf Comput Sci 32(4):373\u2013378","journal-title":"J Chem Inf Comput Sci"},{"issue":"8","key":"823_CR6","doi-asserted-by":"publisher","first-page":"2380","DOI":"10.1021\/ci5002197","volume":"54","author":"P Frasconi","year":"2014","unstructured":"Frasconi P, Gabbrielli F, Lippi M, Marinai S (2014) Markov logic networks for optical chemical structure recognition. J Chem Inf Model 54(8):2380\u20132390","journal-title":"J Chem Inf Model"},{"key":"823_CR7","doi-asserted-by":"crossref","unstructured":"Casey R, Boyer S, Healey P, Miller A, Oudot B, Zilles K (1993) Optical recognition of chemical graphics. In: Proceedings of 2nd international conference on document analysis and recognition (ICDAR'93). IEEE, pp 627\u2013631","DOI":"10.1109\/ICDAR.1993.395658"},{"issue":"3","key":"823_CR8","doi-asserted-by":"publisher","first-page":"740","DOI":"10.1021\/ci800067r","volume":"49","author":"IV Filippov","year":"2009","unstructured":"Filippov IV, Nicklaus MC (2009) Optical structure recognition software to recover chemical information: OSRA, an open source solution. J Chem Inf Model 49(3):740\u2013743","journal-title":"J Chem Inf Model"},{"key":"823_CR9","doi-asserted-by":"crossref","unstructured":"Algorri M-E, Zimmermann M, Friedrich CM, Akle S, Hofmann-Apitius M (2007) Reconstruction of chemical molecules from images. In: 2007 29th annual international conference of the IEEE engineering in medicine and biology society. IEEE, pp 4609\u20134612","DOI":"10.1109\/IEMBS.2007.4353366"},{"key":"823_CR10","doi-asserted-by":"publisher","unstructured":"Rajan K, Brinkhaus HO, Agea MI, Zielesny A, Steinbeck C (2023) DECIMER. ai-An open platform for automated optical chemical structure identification, segmentation and recognition in scientific publications. Nat Commun 14(5045):1-18. https:\/\/doi.org\/10.1038\/s41467-023-40782-0","DOI":"10.1038\/s41467-023-40782-0"},{"issue":"42","key":"823_CR11","doi-asserted-by":"publisher","first-page":"14174","DOI":"10.1039\/D1SC01839F","volume":"12","author":"D-A Clevert","year":"2021","unstructured":"Clevert D-A, Le T, Winter R, Montanari F (2021) Img2Mol\u2014accurate SMILES recognition from molecular graphical depictions. Chem Sci 12(42):14174\u201314181","journal-title":"Chem Sci"},{"issue":"1","key":"823_CR12","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1186\/s13321-022-00624-5","volume":"14","author":"Z Xu","year":"2022","unstructured":"Xu Z, Li J, Yang Z, Li S, Li H (2022) SwinOCSR: end-to-end optical chemical structure recognition using a Swin transformer. J Cheminform 14(1):1\u201313","journal-title":"J Cheminform"},{"issue":"4","key":"823_CR13","doi-asserted-by":"publisher","first-page":"2059","DOI":"10.1021\/acs.jcim.0c00042","volume":"60","author":"EJ Beard","year":"2020","unstructured":"Beard EJ, Cole JM (2020) ChemSchematicResolver: a toolkit to decode 2D chemical diagrams with labels and R-groups into annotated chemical named entities. J Chem Inf Model 60(4):2059\u20132072","journal-title":"J Chem Inf Model"},{"key":"823_CR14","doi-asserted-by":"crossref","unstructured":"Yoo S, Kwon O, Lee H (2022) Image-to-graph transformers for chemical structure recognition. In: ICASSP 2022\u20132022 IEEE international conference on acoustics, speech and signal processing (ICASSP). IEEE, pp 3393\u20133397","DOI":"10.1109\/ICASSP43922.2022.9746088"},{"issue":"3","key":"823_CR15","doi-asserted-by":"publisher","first-page":"1017","DOI":"10.1021\/acs.jcim.8b00669","volume":"59","author":"J Staker","year":"2019","unstructured":"Staker J, Marshall K, Abel R, McQuaw CM (2019) Molecular structure extraction from documents using deep learning. J Chem Inf Model 59(3):1017\u20131029","journal-title":"J Chem Inf Model"},{"issue":"7","key":"823_CR16","doi-asserted-by":"publisher","first-page":"1925","DOI":"10.1021\/acs.jcim.2c01480","volume":"63","author":"Y Qian","year":"2023","unstructured":"Qian Y, Guo J, Tu Z, Li Z, Coley CW, Barzilay RJ (2023) MolScribe: robust molecular structure recognition with image-to-graph generation. J Chem Inf Model 63(7):1925\u20131934","journal-title":"J Chem Inf Model"},{"key":"823_CR17","doi-asserted-by":"crossref","unstructured":"Ronneberger O, Fischer P, Brox T (2015) U-net: convolutional networks for biomedical image segmentation. In: Medical image computing and computer-assisted intervention\u2014MICCAI 2015: 18th international conference, Munich, Germany, October 5\u20139, 2015, proceedings, part III 18. Springer, pp 234\u2013241","DOI":"10.1007\/978-3-319-24574-4_28"},{"key":"823_CR18","doi-asserted-by":"publisher","first-page":"1","DOI":"10.1186\/s13321-021-00496-1","volume":"13","author":"K Rajan","year":"2021","unstructured":"Rajan K, Brinkhaus HO, Sorokina M, Zielesny A, Steinbeck C (2021) DECIMER-segmentation: automated extraction of chemical structure depictions from scientific literature. J Cheminform 13:1\u20139","journal-title":"J Cheminform"},{"key":"823_CR19","doi-asserted-by":"crossref","unstructured":"He K, Gkioxari G, Doll\u00e1r P, Girshick R (2017) Mask r-cnn. In: Proceedings of the IEEE international conference on computer vision, pp 2961\u20132969","DOI":"10.1109\/ICCV.2017.322"},{"issue":"22","key":"823_CR20","doi-asserted-by":"publisher","first-page":"5321","DOI":"10.1021\/acs.jcim.2c00733","volume":"62","author":"Y Xu","year":"2022","unstructured":"Xu Y, Xiao J, Chou C-H, Zhang J, Zhu J, Hu Q, Li H, Han N, Liu B, Zhang S et al (2022) MolMiner: you only look once for chemical structure recognition. J Chem Inf Model 62(22):5321\u20135328","journal-title":"J Chem Inf Model"},{"key":"823_CR21","unstructured":"ChemSAM project. https:\/\/github.com\/mindrank-ai\/ChemSAM\/tree\/master"},{"key":"823_CR22","doi-asserted-by":"crossref","unstructured":"Kirillov A, Mintun E, Ravi N, Mao H, Rolland C, Gustafson L, Xiao T, Whitehead S, Berg AC, Lo W-Y (2023) Segment anything. arXiv:2304.02643","DOI":"10.1109\/ICCV51070.2023.00371"},{"key":"823_CR23","unstructured":"Paszke A, Gross S, Massa F, Lerer A, Bradbury J, Chanan G, Killeen T, Lin Z, Gimelshein N, Antiga L (2019) Pytorch: an imperative style, high-performance deep learning library. In: Advances in neural information processing systems, 32"},{"key":"823_CR24","unstructured":"Dosovitskiy A, Beyer L, Kolesnikov A, Weissenborn D, Zhai X, Unterthiner T, Dehghani M, Minderer M, Heigold G, Gelly S (2020) An image is worth 16x16 words: transformers for image recognition at scale. arXiv:2010.11929"},{"key":"823_CR25","doi-asserted-by":"crossref","unstructured":"He K, Chen X, Xie S, Li Y, Doll\u00e1r P, Girshick R (2022) Masked autoencoders are scalable vision learners. In: Proceedings of the IEEE\/CVF conference on computer vision and pattern recognition, pp 16000\u201316009","DOI":"10.1109\/CVPR52688.2022.01553"},{"key":"823_CR26","first-page":"7537","volume":"33","author":"M Tancik","year":"2020","unstructured":"Tancik M, Srinivasan P, Mildenhall B, Fridovich-Keil S, Raghavan N, Singhal U, Ramamoorthi R, Barron J, Ng R (2020) Fourier features let networks learn high frequency functions in low dimensional domains. Adv Neural Inf Process Syst 33:7537\u20137547","journal-title":"Adv Neural Inf Process Syst"},{"key":"823_CR27","unstructured":"Radford A, Kim JW, Hallacy C, Ramesh A, Goh G, Agarwal S, Sastry G, Askell A, Mishkin P, Clark J (2021) Learning transferable visual models from natural language supervision. In: International conference on machine learning. PMLR, pp 8748\u20138763"},{"key":"823_CR28","doi-asserted-by":"crossref","unstructured":"Carion N, Massa F, Synnaeve G, Usunier N, Kirillov A, Zagoruyko S (2020) End-to-end object detection with transformers. In: European conference on computer vision. Springer, pp 213\u2013229","DOI":"10.1007\/978-3-030-58452-8_13"},{"key":"823_CR29","volume-title":"Advances in neural information processing systems","author":"A Vaswani","year":"2017","unstructured":"Vaswani A, Shazeer N, Parmar N, Uszkoreit J, Jones L, Gomez AN, Kaiser \u0141, Polosukhin I (2017) Attention is all you need. Advances in neural information processing systems, vol 30. MIT Press, Cambridge"},{"key":"823_CR30","unstructured":"Nair V, Hinton GE (2010) Rectified linear units improve restricted Boltzmann machines. In: Proceedings of the 27th international conference on machine learning (ICML-10), pp 807\u2013814"},{"key":"823_CR31","doi-asserted-by":"crossref","unstructured":"Marco AC, Myers A, Graham SJ, D'Agostino P, Apple K (2015) The USPTO patent assignment dataset: descriptions and analysis","DOI":"10.2139\/ssrn.2636461"},{"issue":"11","key":"823_CR32","doi-asserted-by":"publisher","first-page":"2324","DOI":"10.1021\/acs.jcim.5b00559","volume":"55","author":"T Sterling","year":"2015","unstructured":"Sterling T, Irwin JJ (2015) ZINC 15\u2013ligand discovery for everyone. J Chem Inf Model 55(11):2324\u20132337","journal-title":"J Chem Inf Model"},{"key":"823_CR33","unstructured":"Landrum G, et al. RDKit: open-source cheminformatics software. 2016. https:\/\/www.rdkit.org\/, https:\/\/github.com\/rdkit\/rdkit. Accessed 16 October 2023"},{"key":"823_CR34","doi-asserted-by":"crossref","unstructured":"Galamhos C, Matas J, Kittler J (1999) Progressive probabilistic Hough transform for line detection. In: Proceedings 1999 IEEE computer society conference on computer vision and pattern recognition (Cat No PR00149). IEEE, pp 554\u2013560","DOI":"10.1109\/CVPR.1999.786993"},{"key":"823_CR35","doi-asserted-by":"publisher","first-page":"465","DOI":"10.1007\/s13659-021-00305-0","volume":"11","author":"Y-H Wang","year":"2021","unstructured":"Wang Y-H (2021) Bioprospecting: traditional uses and pharmacologically active constituents of Dendrobium plants for dermatological disorders: a review. Nat Prod Bioprospect 11:465\u2013487","journal-title":"Nat Prod Bioprospect"},{"key":"823_CR36","unstructured":"Dataset. https:\/\/drive.google.com\/file\/d\/1RZBpDk4EkM7UI9QDV5gdP2x2iVmqtlR5\/view?usp=drive_link"},{"key":"823_CR37","unstructured":"PROTACT database project. http:\/\/newblock.xq200.com"}],"container-title":["Journal of Cheminformatics"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1186\/s13321-024-00823-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1186\/s13321-024-00823-2\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1186\/s13321-024-00823-2.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,3,13]],"date-time":"2024-03-13T00:05:45Z","timestamp":1710288345000},"score":1,"resource":{"primary":{"URL":"https:\/\/jcheminf.biomedcentral.com\/articles\/10.1186\/s13321-024-00823-2"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,3,12]]},"references-count":37,"journal-issue":{"issue":"1","published-online":{"date-parts":[[2024,12]]}},"alternative-id":["823"],"URL":"https:\/\/doi.org\/10.1186\/s13321-024-00823-2","relation":{},"ISSN":["1758-2946"],"issn-type":[{"value":"1758-2946","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,3,12]]},"assertion":[{"value":"21 November 2023","order":1,"name":"received","label":"Received","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"3 March 2024","order":2,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"12 March 2024","order":3,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"The authors declare no competing interests.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Competing interests"}}],"article-number":"29"}}