<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE article PUBLIC "-//NLM//DTD JATS (Z39.96) Journal Archiving and Interchange DTD with MathML3 v1.2 20190208//EN"  "JATS-archivearticle1-mathml3.dtd"><article xmlns:ali="http://www.niso.org/schemas/ali/1.0/" xmlns:xlink="http://www.w3.org/1999/xlink" article-type="research-article" dtd-version="1.2"><front><journal-meta><journal-id journal-id-type="nlm-ta">elife</journal-id><journal-id journal-id-type="publisher-id">eLife</journal-id><journal-title-group><journal-title>eLife</journal-title></journal-title-group><issn publication-format="electronic" pub-type="epub">2050-084X</issn><publisher><publisher-name>eLife Sciences Publications, Ltd</publisher-name></publisher></journal-meta><article-meta><article-id pub-id-type="publisher-id">84042</article-id><article-id pub-id-type="doi">10.7554/eLife.84042</article-id><article-categories><subj-group subj-group-type="display-channel"><subject>Tools and Resources</subject></subj-group><subj-group subj-group-type="heading"><subject>Neuroscience</subject></subj-group></article-categories><title-group><article-title>Scratch-AID, a deep learning-based system for automatic detection of mouse scratching behavior with high accuracy</article-title></title-group><contrib-group><contrib contrib-type="author" corresp="yes" equal-contrib="yes" id="author-295749"><name><surname>Yu</surname><given-names>Huasheng</given-names></name><contrib-id authenticated="true" contrib-id-type="orcid">https://orcid.org/0000-0001-5641-2512</contrib-id><email>huasheng.yu@pennmedicine.upenn.edu</email><xref ref-type="aff" rid="aff1">1</xref><xref ref-type="fn" rid="equal-contrib1">†</xref><xref ref-type="fn" rid="con1"/><xref ref-type="fn" rid="conf1"/></contrib><contrib contrib-type="author" equal-contrib="yes" id="author-295750"><name><surname>Xiong</surname><given-names>Jingwei</given-names></name><xref ref-type="aff" rid="aff2">2</xref><xref ref-type="fn" rid="equal-contrib1">†</xref><xref ref-type="fn" rid="con2"/><xref ref-type="fn" rid="conf1"/></contrib><contrib contrib-type="author" id="author-295751"><name><surname>Ye</surname><given-names>Adam Yongxin</given-names></name><xref ref-type="aff" rid="aff3">3</xref><xref ref-type="fn" rid="con3"/><xref ref-type="fn" rid="conf1"/></contrib><contrib contrib-type="author" id="author-295752"><name><surname>Cranfill</surname><given-names>Suna Li</given-names></name><contrib-id authenticated="true" contrib-id-type="orcid">https://orcid.org/0000-0002-3431-0061</contrib-id><xref ref-type="aff" rid="aff1">1</xref><xref ref-type="fn" rid="con4"/><xref ref-type="fn" rid="conf1"/></contrib><contrib contrib-type="author" id="author-295753"><name><surname>Cannonier</surname><given-names>Tariq</given-names></name><xref ref-type="aff" rid="aff1">1</xref><xref ref-type="fn" rid="con5"/><xref ref-type="fn" rid="conf1"/></contrib><contrib contrib-type="author" id="author-295754"><name><surname>Gautam</surname><given-names>Mayank</given-names></name><contrib-id authenticated="true" contrib-id-type="orcid">https://orcid.org/0000-0002-7257-5837</contrib-id><xref ref-type="aff" rid="aff1">1</xref><xref ref-type="fn" rid="con6"/><xref ref-type="fn" rid="conf1"/></contrib><contrib contrib-type="author" id="author-295755"><name><surname>Zhang</surname><given-names>Marina</given-names></name><xref ref-type="aff" rid="aff4">4</xref><xref ref-type="fn" rid="con7"/><xref ref-type="fn" rid="conf1"/></contrib><contrib contrib-type="author" id="author-295756"><name><surname>Bilal</surname><given-names>Rayan</given-names></name><xref ref-type="aff" rid="aff1">1</xref><xref ref-type="fn" rid="con8"/><xref ref-type="fn" rid="conf1"/></contrib><contrib contrib-type="author" id="author-295757"><name><surname>Park</surname><given-names>Jong-Eun</given-names></name><xref ref-type="aff" rid="aff1">1</xref><xref ref-type="fn" rid="con9"/><xref ref-type="fn" rid="conf1"/></contrib><contrib contrib-type="author" id="author-295758"><name><surname>Xue</surname><given-names>Yuji</given-names></name><xref ref-type="aff" rid="aff1">1</xref><xref ref-type="fn" rid="con10"/><xref ref-type="fn" rid="conf1"/></contrib><contrib contrib-type="author" id="author-295759"><name><surname>Polam</surname><given-names>Vidhur</given-names></name><xref ref-type="aff" rid="aff1">1</xref><xref ref-type="fn" rid="con11"/><xref ref-type="fn" rid="conf1"/></contrib><contrib contrib-type="author" id="author-295760"><name><surname>Vujovic</surname><given-names>Zora</given-names></name><xref ref-type="aff" rid="aff1">1</xref><xref ref-type="fn" rid="con12"/><xref ref-type="fn" rid="conf1"/></contrib><contrib contrib-type="author" id="author-295761"><name><surname>Dai</surname><given-names>Daniel</given-names></name><xref ref-type="aff" rid="aff1">1</xref><xref ref-type="fn" rid="con13"/><xref ref-type="fn" rid="conf1"/></contrib><contrib contrib-type="author" id="author-295763"><name><surname>Ong</surname><given-names>William</given-names></name><xref ref-type="aff" rid="aff1">1</xref><xref ref-type="fn" rid="con14"/><xref ref-type="fn" rid="conf1"/></contrib><contrib contrib-type="author" id="author-295764"><name><surname>Ip</surname><given-names>Jasper</given-names></name><contrib-id authenticated="true" contrib-id-type="orcid">https://orcid.org/0000-0001-9773-1544</contrib-id><xref ref-type="aff" rid="aff1">1</xref><xref ref-type="fn" rid="con15"/><xref ref-type="fn" rid="conf1"/></contrib><contrib contrib-type="author" id="author-295765"><name><surname>Hsieh</surname><given-names>Amanda</given-names></name><xref ref-type="aff" rid="aff1">1</xref><xref ref-type="fn" rid="con16"/><xref ref-type="fn" rid="conf1"/></contrib><contrib contrib-type="author" id="author-295766"><name><surname>Mimouni</surname><given-names>Nour</given-names></name><xref ref-type="aff" rid="aff1">1</xref><xref ref-type="fn" rid="con17"/><xref ref-type="fn" rid="conf1"/></contrib><contrib contrib-type="author" id="author-295775"><name><surname>Lozada</surname><given-names>Alejandra</given-names></name><xref ref-type="aff" rid="aff1">1</xref><xref ref-type="fn" rid="con18"/><xref ref-type="fn" rid="conf1"/></contrib><contrib contrib-type="author" id="author-233281"><name><surname>Sosale</surname><given-names>Medhini</given-names></name><xref ref-type="aff" rid="aff1">1</xref><xref ref-type="fn" rid="con19"/><xref ref-type="fn" rid="conf1"/></contrib><contrib contrib-type="author" id="author-295773"><name><surname>Ahn</surname><given-names>Alex</given-names></name><xref ref-type="aff" rid="aff1">1</xref><xref ref-type="fn" rid="con20"/><xref ref-type="fn" rid="conf1"/></contrib><contrib contrib-type="author" id="author-97826"><name><surname>Ma</surname><given-names>Minghong</given-names></name><xref ref-type="aff" rid="aff1">1</xref><xref ref-type="fn" rid="con21"/><xref ref-type="fn" rid="conf1"/></contrib><contrib contrib-type="author" id="author-108107"><name><surname>Ding</surname><given-names>Long</given-names></name><contrib-id authenticated="true" contrib-id-type="orcid">https://orcid.org/0000-0002-1716-3848</contrib-id><xref ref-type="aff" rid="aff1">1</xref><xref ref-type="other" rid="fund3"/><xref ref-type="fn" rid="con22"/><xref ref-type="fn" rid="conf1"/></contrib><contrib contrib-type="author" id="author-295777"><name><surname>Arsuaga</surname><given-names>Javier</given-names></name><xref ref-type="aff" rid="aff5">5</xref><xref ref-type="aff" rid="aff6">6</xref><xref ref-type="other" rid="fund1"/><xref ref-type="fn" rid="con23"/><xref ref-type="fn" rid="conf1"/></contrib><contrib contrib-type="author" corresp="yes" id="author-18016"><name><surname>Luo</surname><given-names>Wenqin</given-names></name><contrib-id authenticated="true" contrib-id-type="orcid">https://orcid.org/0000-0002-2486-807X</contrib-id><email>luow@pennmedicine.upenn.edu</email><xref ref-type="aff" rid="aff1">1</xref><xref ref-type="other" rid="fund2"/><xref ref-type="other" rid="fund3"/><xref ref-type="fn" rid="con24"/><xref ref-type="fn" rid="conf1"/></contrib><aff id="aff1"><label>1</label><institution-wrap><institution-id institution-id-type="ror">https://ror.org/00b30xv10</institution-id><institution>Department of Neuroscience, Perelman School of Medicine, University of Pennsylvania</institution></institution-wrap><addr-line><named-content content-type="city">Philadelphia</named-content></addr-line><country>United States</country></aff><aff id="aff2"><label>2</label><institution-wrap><institution-id institution-id-type="ror">https://ror.org/05rrcem69</institution-id><institution>Graduate Group in Biostatistics, University of California Davis</institution></institution-wrap><addr-line><named-content content-type="city">Davis</named-content></addr-line><country>United States</country></aff><aff id="aff3"><label>3</label><institution-wrap><institution-id institution-id-type="ror">https://ror.org/006w34k90</institution-id><institution>Howard Hughes Medical Institute, Program in Cellular and Molecular Medicine, Boston Children’s Hospital, Harvard Medical School</institution></institution-wrap><addr-line><named-content content-type="city">Boston</named-content></addr-line><country>United States</country></aff><aff id="aff4"><label>4</label><institution-wrap><institution-id institution-id-type="ror">https://ror.org/042nb2s44</institution-id><institution>Department of Electrical Engineering and Computer Science, Massachusetts Institute of Technology</institution></institution-wrap><addr-line><named-content content-type="city">Cambridge</named-content></addr-line><country>United States</country></aff><aff id="aff5"><label>5</label><institution-wrap><institution-id institution-id-type="ror">https://ror.org/05rrcem69</institution-id><institution>Department of Molecular and Cellular Biology, University of California Davis</institution></institution-wrap><addr-line><named-content content-type="city">Davis</named-content></addr-line><country>United States</country></aff><aff id="aff6"><label>6</label><institution-wrap><institution-id institution-id-type="ror">https://ror.org/05rrcem69</institution-id><institution>Department of Mathematics, University of California Davis</institution></institution-wrap><addr-line><named-content content-type="city">Davis</named-content></addr-line><country>United States</country></aff></contrib-group><contrib-group content-type="section"><contrib contrib-type="editor"><name><surname>Kim</surname><given-names>Brian S</given-names></name><role>Reviewing Editor</role><aff><institution-wrap><institution-id institution-id-type="ror">https://ror.org/04a9tmd77</institution-id><institution>Icahn School of Medicine at Mount Sinai</institution></institution-wrap><country>United States</country></aff></contrib><contrib contrib-type="senior_editor"><name><surname>Dulac</surname><given-names>Catherine</given-names></name><role>Senior Editor</role><aff><institution-wrap><institution-id institution-id-type="ror">https://ror.org/03vek6s52</institution-id><institution>Harvard University</institution></institution-wrap><country>United States</country></aff></contrib></contrib-group><author-notes><fn fn-type="con" id="equal-contrib1"><label>†</label><p>These authors contributed equally to this work</p></fn></author-notes><pub-date publication-format="electronic" date-type="publication"><day>08</day><month>12</month><year>2022</year></pub-date><pub-date pub-type="collection"><year>2022</year></pub-date><volume>11</volume><elocation-id>e84042</elocation-id><history><date date-type="received" iso-8601-date="2022-10-09"><day>09</day><month>10</month><year>2022</year></date><date date-type="accepted" iso-8601-date="2022-11-29"><day>29</day><month>11</month><year>2022</year></date></history><pub-history><event><event-desc>This manuscript was published as a preprint at bioRxiv.</event-desc><date date-type="preprint" iso-8601-date="2022-10-07"><day>07</day><month>10</month><year>2022</year></date><self-uri content-type="preprint" xlink:href="https://doi.org/10.1101/2022.10.07.511352"/></event></pub-history><permissions><copyright-statement>© 2022, Yu, Xiong et al</copyright-statement><copyright-year>2022</copyright-year><copyright-holder>Yu, Xiong et al</copyright-holder><ali:free_to_read/><license xlink:href="http://creativecommons.org/licenses/by/4.0/"><ali:license_ref>http://creativecommons.org/licenses/by/4.0/</ali:license_ref><license-p>This article is distributed under the terms of the <ext-link ext-link-type="uri" xlink:href="http://creativecommons.org/licenses/by/4.0/">Creative Commons Attribution License</ext-link>, which permits unrestricted use and redistribution provided that the original author and source are credited.</license-p></license></permissions><self-uri content-type="pdf" xlink:href="elife-84042-v2.pdf"/><self-uri content-type="figures-pdf" xlink:href="elife-84042-figures-v2.pdf"/><abstract><p>Mice are the most commonly used model animals for itch research and for development of anti-itch drugs. Most laboratories manually quantify mouse scratching behavior to assess itch intensity. This process is labor-intensive and limits large-scale genetic or drug screenings. In this study, we developed a new system, Scratch-AID (<bold><underline>A</underline></bold>utomatic <bold><underline>I</underline></bold>tch <bold><underline>D</underline></bold>etection), which could automatically identify and quantify mouse scratching behavior with high accuracy. Our system included a custom-designed videotaping box to ensure high-quality and replicable mouse behavior recording and a convolutional recurrent neural network trained with frame-labeled mouse scratching behavior videos, induced by nape injection of chloroquine. The best trained network achieved 97.6% recall and 96.9% precision on previously unseen test videos. Remarkably, Scratch-AID could reliably identify scratching behavior in other major mouse itch models, including the acute cheek model, the histaminergic model, and a chronic itch model. Moreover, our system detected significant differences in scratching behavior between control and mice treated with an anti-itch drug. Taken together, we have established a novel deep learning-based system that could replace manual quantification for mouse scratching behavior in different itch models and for drug screening.</p></abstract><kwd-group kwd-group-type="author-keywords"><kwd>itch</kwd><kwd>mouse behavior</kwd><kwd>scratching</kwd><kwd>deep learning</kwd><kwd>automatic quantification</kwd></kwd-group><kwd-group kwd-group-type="research-organism"><title>Research organism</title><kwd>Mouse</kwd></kwd-group><funding-group><award-group id="fund1"><funding-source><institution-wrap><institution-id institution-id-type="FundRef">http://dx.doi.org/10.13039/100000001</institution-id><institution>National Science Foundation</institution></institution-wrap></funding-source><award-id>DMS-1854770</award-id><principal-award-recipient><name><surname>Arsuaga</surname><given-names>Javier</given-names></name></principal-award-recipient></award-group><award-group id="fund2"><funding-source><institution-wrap><institution-id institution-id-type="FundRef">http://dx.doi.org/10.13039/100000002</institution-id><institution>National Institutes of Health</institution></institution-wrap></funding-source><award-id>R01 NS083702</award-id><principal-award-recipient><name><surname>Luo</surname><given-names>Wenqin</given-names></name></principal-award-recipient></award-group><award-group id="fund3"><funding-source><institution-wrap><institution-id institution-id-type="FundRef">http://dx.doi.org/10.13039/100000002</institution-id><institution>National Institutes of Health</institution></institution-wrap></funding-source><award-id>R34 NS118411</award-id><principal-award-recipient><name><surname>Ding</surname><given-names>Long</given-names></name><name><surname>Luo</surname><given-names>Wenqin</given-names></name></principal-award-recipient></award-group><funding-statement>The funders had no role in study design, data collection, and interpretation, or the decision to submit the work for publication.</funding-statement></funding-group><custom-meta-group><custom-meta specific-use="meta-only"><meta-name>Author impact statement</meta-name><meta-value>Scratch-AID, a deep learning-based system for automatic quantification of mouse scratching behavior, could replace labor-intensive manual quantification and facilitate high through-put anti-itch drug screening.</meta-value></custom-meta></custom-meta-group></article-meta></front><body><sec id="s1" sec-type="intro"><title>Introduction</title><p>Itch is a disturbing symptom associated with skin diseases, immune problems, systemic diseases, and mental disorders (<xref ref-type="bibr" rid="bib6">Cevikbas and Lerner, 2020</xref>; <xref ref-type="bibr" rid="bib16">Hong et al., 2011</xref>; <xref ref-type="bibr" rid="bib19">Kremer et al., 2020</xref>; <xref ref-type="bibr" rid="bib35">Ständer et al., 2007</xref>). Chronic itch affects about 13–17% of the population (<xref ref-type="bibr" rid="bib24">Matterne et al., 2009</xref>; <xref ref-type="bibr" rid="bib38">Weisshaar and Dalgard, 2009</xref>), severely worsening the quality of life of affected patients. Unfortunately, treatment options for many chronic itch conditions are still limited (<xref ref-type="bibr" rid="bib41">Yosipovitch et al., 2018</xref>; <xref ref-type="bibr" rid="bib42">Yu et al., 2021</xref>).</p><p>Mice are the most widely used model animals for studying itch mechanisms and for developing new preclinical anti-itch drugs (<xref ref-type="bibr" rid="bib14">Han et al., 2013</xref>; <xref ref-type="bibr" rid="bib20">Liu et al., 2009</xref>; <xref ref-type="bibr" rid="bib34">Solinski et al., 2019</xref>; <xref ref-type="bibr" rid="bib36">Sun and Chen, 2007</xref>). Since itch is an unpleasant sensation that provokes the desire to scratch (<xref ref-type="bibr" rid="bib17">Ikoma et al., 2006</xref>), scratching behavior has been assessed as a proxy for itch intensity in mice (<xref ref-type="bibr" rid="bib20">Liu et al., 2009</xref>; <xref ref-type="bibr" rid="bib25">Morita et al., 2015</xref>). Till now, this quantification process has been mainly conducted by watching videos and manually counting scratching bouts or the total scratching time, which is tedious and time consuming, unavoidably introduces human errors and bias, and limits the large-scale genetic or drug screenings.</p><p>Given the biological importance and the obvious need, several research groups have tried different strategies to automate this process, including an acoustic recording method (<xref ref-type="bibr" rid="bib10">Elliott et al., 2017</xref>), a method using magnetic field and metal ring to detect paw movement (<xref ref-type="bibr" rid="bib26">Mu et al., 2017</xref>), and several video analysis-based approaches (<xref ref-type="bibr" rid="bib4">Bohnslav et al., 2021</xref>; <xref ref-type="bibr" rid="bib18">Kobayashi et al., 2021</xref>; <xref ref-type="bibr" rid="bib27">Park et al., 2019</xref>; <xref ref-type="bibr" rid="bib31">Sakamoto et al., 2022</xref>). Nevertheless, these methods have not been widely adopted by other research laboratories, due to the uncertain performance of the trained models in different lab environments, the requirement of specialized equipment, and/or inadequate evaluation of these methods in different mouse itch models.</p><p>In recent years, with the rapid progress in the field of artificial intelligence, deep learning has been applied in various scientific research areas. For example, convolutional neural networks (CNN) are widely used in computer visual recognition tasks (<xref ref-type="bibr" rid="bib12">Gu et al., 2018</xref>), whereas recurrent neural networks (RNN) (<xref ref-type="bibr" rid="bib11">Graves, 2013</xref>) are developed for analyzing temporal dynamic features. Moreover, rapid improvement of computing power, especially in the graphics processing unit capacity, together with new open-source deep learning libraries, such as PyTorch (<xref ref-type="bibr" rid="bib28">Paszke et al., 2019</xref>), Keras (<xref ref-type="bibr" rid="bib13">Gulli and Pal, 2017</xref>), and Tensorflow (<xref ref-type="bibr" rid="bib1">Abadi et al., 2016</xref>), have greatly accelerated broad applications of deep learning.</p><p>Animal behavior analysis is one of the research areas benefiting from the applications of deep learning. For example, DeepLabCut can track different body parts in freely moving animals for behavior analysis (<xref ref-type="bibr" rid="bib23">Mathis et al., 2018</xref>). DeepEthogram recognizes and annotates different behaviors of mice and flies (<xref ref-type="bibr" rid="bib4">Bohnslav et al., 2021</xref>). These examples support the proof-of-principle that deep-learning is a powerful avenue for automating animal behavior analysis. Nevertheless, for a given animal behavior, like mouse scratching, a designated method, which achieves high sensitivity, specificity, and generalization to replace human observers, still needs to be established.</p><p>To meet this challenge, we developed a new deep learning-based system, Scratch-AID (<bold><underline>A</underline></bold>utomatic <bold><underline>I</underline></bold>tch <bold><underline>D</underline></bold>etection), which achieved automatic quantification of mouse scratching behavior with high accuracy. We first designed a videotaping box to acquire high-quality recording of mouse behavior in a reproducible environment from the bottom. We recorded 40 videos of 10 adult wildtype mice (5 males and 5 females) after nape injection of a non-histamine pruritogen, chloroquine (CQ), and manually labeled all video frames as the reference. We then designed a convolutional recurrent neural network (CRNN) by combining CNN and RNN and trained it with 32 scratching videos from 8 randomly picked mice. We obtained a series of prediction models using different training parameters and evaluated these models with test videos (eight unseen test videos from the two remaining mice). The best trained model achieved 97.6% recall and 96.9% precision on test videos, similar to the manual quantification results. Impressively, Scratch-AID could also quantify scratching behavior from other major acute and chronic itch models with high accuracy. Lastly, we applied Scratch-AID in an anti-itch drug screening paradigm and found that it reliably detected the drug effect. In summary, we have established a new system for accurate automatic quantification of mouse scratching behavior. Based on the performances, Scratch-AID could replace manual quantification for various mouse itch models and for drug or genetic screenings.</p></sec><sec id="s2" sec-type="results"><title>Results</title><sec id="s2-1"><title>The overall workflow</title><p>Our workflow to develop a new system for detection and quantification of mouse scratching behavior consists of four major steps (<xref ref-type="fig" rid="fig1">Figure 1A</xref>): (1) Videotape mouse scratching behavior induced by an acute nape itch model; (2) Manually annotate scratching frames in all videos for training and test datasets; (3) Design a deep learning neural network; train this network with randomly selected training videos and adjust different training parameters; and evaluate the performance of the trained neural networks on test videos; (4) Evaluate the generalization of the trained neural network in additional itch models and a drug screening paradigm.</p><fig-group><fig id="fig1" position="float"><label>Figure 1.</label><caption><title>The overall workflow and building a customized videotaping box for mouse scratching behavior recording.</title><p>(<bold>A</bold>) A diagram showing the workflow to develop a deep learning-based system for automatic detection and quantification of mouse scratching behavior. (<bold>B</bold>) An image of the designed videotaping box for high-quality video recording of mouse scratching behavior. Scale bar, 5 cm. (<bold>C</bold>) A cartoon showing the acute itch model induced by the chloroquine (CQ) injection in the nape, followed by video recording in the customized videotaping box. (<bold>D</bold>) Representative images showing different phases (P1–P4) of a scratching train (upper). Red arrows indicate the scratching hind paw. A cartoon showing the dynamic movement of the scratching hind paw in a scratching train (bottom). The cycle of scratching bout (P2) and paw licking (P3) may repeat once or more times in a scratching train. Scale bar, 1 cm. (<bold>E</bold>) The total number of scratching trains in each video. (<bold>F</bold>) The distribution of scratching train duration (<italic>n</italic> = 1135 scratching trains). The inset is the zoom-in of the red rectangle.</p></caption><graphic mimetype="image" mime-subtype="tiff" xlink:href="elife-84042-fig1-v2.tif"/></fig><fig id="fig1s1" position="float" specific-use="child-fig"><label>Figure 1—figure supplement 1.</label><caption><title>Dynamic and static features of scratching behavior in the chloroquine (CQ) nape acute itch model.</title><p>(<bold>A, B</bold>) The scratching hind paw (red arrow in A) but not the contralateral hind paw displayed rhythmic vibration during scratching behavior (<bold>B</bold>). The distance between scratching hind paw and nose (<bold>C</bold>) or mouth (<bold>D</bold>), the angle consisting of nose, scratching hind paw, and ipsilateral front paw (<bold>E</bold>), and the angle consisting of scratching hind paw, mouth, and tail (<bold>F</bold>) in the scratching or non-scratching frames (<italic>N</italic> = 1756 frames for non-scratching, 2214 frames for scratching). Error bar, standard error of the mean (SEM). Differences between the two groups were analyzed using unpaired two-tailed Student’s <italic>t</italic>-test,*** p &lt; 0.001.</p></caption><graphic mimetype="image" mime-subtype="tiff" xlink:href="elife-84042-fig1-figsupp1-v2.tif"/></fig></fig-group></sec><sec id="s2-2"><title>Design a videotaping box for high-quality and reproducible recording of mouse scratching behavior</title><p>High-quality videos recorded from a reproducible environment are critical for stable performance of trained prediction models and for adoption by other research laboratories. Thus, we designed a mouse videotaping box for such a purpose. It consisted of two boxes with white acrylic walls joined by a transparent acrylic floor (<xref ref-type="fig" rid="fig1">Figure 1B</xref>). The top (‘mouse’) box (Length × Width × Height = 14.68 × 14.68 × 5 cm) had a lid for mouse access. The bottom (‘camera’) box (Length × Width × Height = 14.68 × 14.68 × 23.6 cm) had a door for access to the camera (Logitech C920e Business Webcam). The walls and the lid of the box were non-transparent to minimize interference from outside visual stimuli. Ambient light penetrated the walls and provided sufficient illumination for behavior recording. A mouse could freely move inside the top box, and a camera recorded mouse behavior from the bottom (30 frames/s). Compared to the top or side views, the bottom view can clearly capture the key body parts involved in scratching behavior, such as the scratching hind paw and mouth, and their movements in great details (<xref ref-type="video" rid="video1">Video 1</xref>). The magnification, resolution, and brightness of the video can be adjusted by the camera recording software (Logitech C920e Business Webcam driver and software) to achieve consistent video recording. In short, this customized videotaping box allows high-quality video recording of mouse scratching and other behaviors in a stable and reproducible environment.</p><media mimetype="video" mime-subtype="mp4" xlink:href="elife-84042-video1.mp4" id="video1"><label>Video 1.</label><caption><title>An example of a mouse scratching train recorded by the designed videotaping box.</title></caption></media><p>Spontaneous scratching is a rare event under normal conditions in mice. Itch sensation and scratching are usually induced by different itch models for research. Common mouse itch models are classified as cheek or nape, histaminergic or non-histaminergic, and acute or chronic, based on the body location where itch sensation is evoked, the kind of pruritogens, and the duration of itch sensation (<xref ref-type="bibr" rid="bib17">Ikoma et al., 2006</xref>; <xref ref-type="bibr" rid="bib21">Liu and Dong, 2015</xref>; <xref ref-type="bibr" rid="bib33">Shimada and LaMotte, 2008</xref>; <xref ref-type="bibr" rid="bib37">Thurmond et al., 2008</xref>). We first used an acute nape itch model induced by a non-histaminergic pruritogen, CQ, because it triggered immediate and robust scratching behavior in mice (<xref ref-type="bibr" rid="bib20">Liu et al., 2009</xref>). After intradermal injection of CQ (200 μg in 15 μl saline) into left or right nape of the mice, a 20-min video was recorded using the customized videotaping box (<xref ref-type="fig" rid="fig1">Figure 1C</xref>).</p><p>In response to CQ injection, mice scratched the affected skin region using their ipsilateral hind paw. Evoked mice displayed multiple episodes of scratching, which were separated by non-scratching phases. Each episode of scratching, here defined as a scratching train, usually contained four phases: start (lifting the scratching hind paw toward the affected skin), scratching bout (rhythmic movement of hind paw against the affected skin), paw licking (putting the scratching hind paw into the mouth and licking), and end (putting down the scratching hind paw back to the floor) (<xref ref-type="fig" rid="fig1">Figure 1D</xref> and <xref ref-type="video" rid="video1">Video 1</xref>). The cycle of scratching bout and paw licking might occur once or repeat multiple times, depending on the itch intensity and the internal state of the mouse. The time from the start to the end of a given scratching train is defined as the duration. The total scratching time is the sum of durations of all scratching trains, which is an effective parameter to quantify the scratching behavior and assess itch intensity.</p></sec><sec id="s2-3"><title>Video annotation</title><p>Forty scratching videos from 10 adult wildtype C57 mice (5 males and 5 females, 2- to 3-month-old) were recorded (<xref ref-type="supplementary-material" rid="supp1">Supplementary file 1</xref>). For neural network training and testing and for comparing performances between trained neural networks and manual quantification, two methods were used to annotate mouse scratching behavior in these videos. The first method (<italic>manual annotation</italic>) was to watch these videos at normal (1×) speed (30 frames/s) and label the start and end time points (converted into frame numbers for subsequent analysis) of each scratching train. This is consistent with the field common practice for manually quantifying mouse scratching behavior. Our manual annotation results were produced by 10 human observers, thus reflecting an averaged precision of the manual quantification process. The second method (<italic>reference annotation</italic>, or ground-truth annotation) was to accurately determine the start and end of each scratching train by analyzing each video frame-by-frame. The reference annotation of the 40 videos were used as the training and test datasets. The total number of scratching train in each video and the distribution of scratching train durations were quantified (<xref ref-type="fig" rid="fig1">Figure 1E, F</xref> and <xref ref-type="supplementary-material" rid="supp2">Supplementary file 2</xref>).</p></sec><sec id="s2-4"><title>Deep learning neural network design and model training</title><p>Mouse scratching behavior displayed unique dynamic (temporal) and static (spatial) features, which were highlighted by tracking the key body parts using DeepLabCut (<xref ref-type="bibr" rid="bib23">Mathis et al., 2018</xref>; <xref ref-type="fig" rid="fig1s1">Figure 1—figure supplement 1</xref>; <xref ref-type="video" rid="video2">Video 2</xref>). One of the most obvious dynamic features was the rhythmic movement of the scratching hind paw (<xref ref-type="fig" rid="fig1s1">Figure 1—figure supplement 1A, B</xref>). Some unique static features included the relative positional relationships between the scratching hind paw and other body parts (<xref ref-type="fig" rid="fig1s1">Figure 1—figure supplement 1C–F</xref>). To fully capture these dynamic and static features, we designed a CRNN to take advantage of the different strengths of CNN and RNN (<xref ref-type="fig" rid="fig2">Figure 2A</xref> and <xref ref-type="fig" rid="fig2s1">Figure 2—figure supplement 1A</xref>). The CRNN contained a CNN (ResNet-18 <xref ref-type="bibr" rid="bib15">He et al., 2016</xref>; <xref ref-type="fig" rid="fig2s1">Figure 2—figure supplement 1B</xref>) that extracts static features, such as the relative position of different body parts, an RNN (two-layer bidirectional gated recurrent unit [GRU]; <xref ref-type="bibr" rid="bib9">Dey and Salem, 2017</xref>; <xref ref-type="fig" rid="fig2s1">Figure 2—figure supplement 1C</xref>) that extracts dynamic features, such as the rhythmic movement of the scratching hind paw in consecutive frames, and a fully connected layer (the classifier) to combine the features extracted by both CNN and RNN and generate the prediction output.</p><media mimetype="video" mime-subtype="mp4" xlink:href="elife-84042-video2.mp4" id="video2"><label>Video 2.</label><caption><title>Key body parts tracked by DeepLabCut showing the dynamic and static features of mouse scratching behavior.</title></caption></media><fig-group><fig id="fig2" position="float"><label>Figure 2.</label><caption><title>Deep learning neural network design and training.</title><p>(<bold>A</bold>) Cartoon showing the architecture of designed deep learning neural network consisting of the combination of convolutional neural networks (CNN), recurrent neural networks (RNN), and classifier. (<bold>B</bold>) Cartoon showing the preparation of inputs for the training dataset. Consecutive <italic>N</italic> frames were selected as one input for training. The interval between two adjacent inputs in a video was 4–10 frames. (<bold>C</bold>) The information of a sample training and test datasets. The training loss decreased (<bold>D</bold>) while the accuracy increased (<bold>E</bold>) during the training process with different input length (<italic>N</italic> = 3, 5, 7, 13, 23, 45 frames). The inset is the zoom-in of part of the figure.</p></caption><graphic mimetype="image" mime-subtype="tiff" xlink:href="elife-84042-fig2-v2.tif"/></fig><fig id="fig2s1" position="float" specific-use="child-fig"><label>Figure 2—figure supplement 1.</label><caption><title>The architecture of deep learning neural network.</title><p>(<bold>A</bold>) The cartoon showing the architecture of designed convolutional recurrent neural network (CRNN). The input was first fed into convolutional neural networks (CNN, ResNet-18), then followed by a two-layer bidirectional gated recurrent unit (GRU). A final full connection (FC) layer generated the binary prediction results. (<bold>B</bold>) Details of the modified ResNet-18 network. The final FC layer was modified to output frame-wise embedding instead of classification result. Conv, convolutional layer; Batch norm, batch normalization layer; ReLU, rectified linear unit; ‘+’, element wise plus for vector. (<bold>C</bold>) Details of one GRU of the two-layer bidirectional GRU. The unit connected to embedding of frame <italic>t</italic>(<italic>x</italic><sub><italic>t</italic></sub>) took the last unit’s output (<italic>h</italic><sub><italic>t</italic>−1</sub>) as input and generated new output (<italic>h</italic><sub><italic>t</italic></sub>). The three yellow squares inside the unit were the reset gate, update gate, and candidate activation vector. ‘<italic>x</italic>’, Hadamard product for vectors; ‘+’, vector plus.</p></caption><graphic mimetype="image" mime-subtype="tiff" xlink:href="elife-84042-fig2-figsupp1-v2.tif"/></fig><fig id="fig2s2" position="float" specific-use="child-fig"><label>Figure 2—figure supplement 2.</label><caption><title>Distribution of durations of scratching bout and paw licking in the chloroquine (CQ) nape acute itch model.</title><p>The frequency distribution of scratching bout (<bold>A</bold>) or paw licking (<bold>B</bold>) durations. The inset figure is the zoom-in of the red square part.</p></caption><graphic mimetype="image" mime-subtype="tiff" xlink:href="elife-84042-fig2-figsupp2-v2.tif"/></fig></fig-group><p>The 40 videos were randomly split into two parts, 80% of them (32 videos from 8 mice) were assigned to the training dataset and 20% of them (8 videos from 2 mice) to the test dataset (<xref ref-type="fig" rid="fig2">Figure 2C</xref>). Each video was converted into individual frames, and each frame was classified as ‘scratching’ (within a scratching train) or ‘non-scratching’ (out of a scratching train) based on the reference annotation. For the training dataset preparation, <italic>N</italic> consecutive frames (a parameter adjusted for optimal model performance) were selected as one input to capture the dynamic features of scratching (<xref ref-type="fig" rid="fig2">Figure 2B</xref>). To avoid large sets of redundancy in the training dataset, two adjacent inputs were apart between 4 and 10 frames (<xref ref-type="fig" rid="fig2">Figure 2B</xref>). An input was labeled as ‘scratching’ (class 1) if more than half of frames (<italic>N</italic>/2) in the input were scratching frames; otherwise labeled as ‘non-scratching’ (class 0) (<xref ref-type="fig" rid="fig2">Figure 2B</xref>).</p><p>Since the dynamic features of scratching behavior spanned multiple frames, the input length (<italic>N</italic> frames) would be a critical training parameter. In CQ triggered acute nape itch model, the average duration of one cycle of scratching bout and paw licking was around 30 frames (<xref ref-type="fig" rid="fig2s2">Figure 2—figure supplement 2A, B</xref>). Thus, we tested a range of input length from 3 to 45 frames for model training. During the training process, the loss (discrepancy between model prediction and reference annotation) decreased, and the prediction accuracy (correct prediction of both scratching and non-scratching frames/all frames) increased (<xref ref-type="fig" rid="fig2">Figure 2D, E</xref>). After 10 epochs (one epoch means the training covers the complete training dataset for one round), the accuracy reached a plateau (<xref ref-type="fig" rid="fig2">Figure 2E</xref>). The prediction accuracies were more than 0.98 for all input length, improving slightly with the increase of the input length (<xref ref-type="fig" rid="fig2">Figure 2E</xref>). These results demonstrate that the designed CRNN network works very efficiently to capture the scratching features and recognize scratching behavior in the training dataset.</p></sec><sec id="s2-5"><title>Model evaluation on test datasets</title><p>We evaluated the performance of the trained prediction models on eight unseen test videos. First, similar to what described above, each test video was converted into inputs with ‘<italic>N</italic>’ frames (the same ‘<italic>N</italic>’ was used for training and test), except that the two adjacent inputs were only 1 frame apart. Second, the trained neural network predicted each input to be ‘scratching’ or ‘non-scratching’. Third, to convert the prediction from one input (containing <italic>N</italic> frames) into the prediction for each individual frame, we used the following rule: the prediction of the middle frame of each input would be the same as that of the input. For example, if an input was predicted as ‘scratching’, then the middle frame of this input would be a ‘scratching’ frame. This conversion predicted each frame of tested videos as ‘scratching’ or ‘non-scratching’ expect for the few frames at the beginning or at the end of a video (see method for the missing data interpretation). Fourth, recall (the number of correctly predicted scratching frames/the number of reference scratching frames), precision (the number of correctly predicted scratching frames/the number of all predicted scratching frames), and <italic>F</italic>1 score (2*recall*precision/(recall + precision)) were calculated. Compared to the overall accuracy (correct prediction of both scratching and non-scratching frames/all frames), the recall, precision, and <italic>F</italic>1 score give a more precise and in depth evaluation of a model’s performance (<xref ref-type="bibr" rid="bib29">Powers, 2020</xref>), especially when scratching is a relative rare event in a video.</p><p>To rule out the possibility that the good performance of our models was due to a specific combination of the training and test datasets, we rotated the training and test videos for cross-validation. For all different combination of training and test videos, <italic>F</italic>1 scores were all above 0.9 (<xref ref-type="fig" rid="fig3s1">Figure 3—figure supplement 1A, B</xref>), supporting the stable and high performance of our prediction models. In addition, the prediction model performed better with the increased input length (<xref ref-type="fig" rid="fig3s1">Figure 3—figure supplement 1C</xref>). The best model, the one trained with videos 1–32 with the input length of 45 (<italic>N</italic> = 45), was selected for additional analyses and tests.</p><p>The average recall and precision of the best model were 97.6% and 96.9%, respectively, for the eight test videos (<xref ref-type="fig" rid="fig3">Figure 3A</xref>), and the recall and precision for individual videos were above 95% in most cases (<xref ref-type="fig" rid="fig3">Figure 3B</xref>). This performance was similar to or even slightly better than that of manual annotation, which had an average recall and precision of 95.1% and 94.2%, respectively (<xref ref-type="fig" rid="fig3">Figure 3C, D</xref>). When comparing the total scratching time to the reference annotation, the average discrepancy of the model prediction was 1.9% whereas that of manual annotation was 2.1% (<xref ref-type="fig" rid="fig3">Figure 3E</xref>). The correlation between the model prediction and the reference annotation was 0.98, similar to the manual annotation results (<xref ref-type="fig" rid="fig3">Figure 3F</xref>). When examining the probability traces of model prediction and the reference annotation (one example shown in <xref ref-type="fig" rid="fig3">Figure 3G</xref>), we found that the model successfully recognized almost all the scratching trains in test videos, and that the prediction of the start and end of each scratching train aligned well with the reference annotation (<xref ref-type="fig" rid="fig3">Figure 3H</xref>). Taken together, these results demonstrate the high reliability and accuracy of our model to recognize and quantify mouse scratching behavior of new videos.</p><fig-group><fig id="fig3" position="float"><label>Figure 3.</label><caption><title>Performance of the best model on test videos.</title><p>The recall, precision, and <italic>F</italic>1 score of the best model on average (<bold>A</bold>) or in individual videos (<bold>B</bold>). The recall, precision, and <italic>F</italic>1 score of manual annotation on average (<bold>C</bold>) or in individual videos (<bold>D</bold>). (<bold>E</bold>) The comparison among model prediction, manual quantification, and the reference annotation. The reference annotation is normalized to 100% shown as the red line. (<bold>F</bold>) The correlations between model prediction or manual quantification and reference annotation. <italic>R</italic><sup>2</sup>, Pearson correlation coefficient. (<bold>G</bold>) An example scratching probability trace (red curve) predicted by the model and aligned with the reference annotation (green bar). (<bold>H</bold>) The two zoom-ins from (<bold>G</bold>) showing the nice alignment between the model prediction and the reference annotation.</p></caption><graphic mimetype="image" mime-subtype="tiff" xlink:href="elife-84042-fig3-v2.tif"/></fig><fig id="fig3s1" position="float" specific-use="child-fig"><label>Figure 3—figure supplement 1.</label><caption><title>Cross-validation and parameter optimization of the prediction models.</title><p>(<bold>A</bold>) Cross-validation of the trained prediction models by rotating the training and test datasets in 40 videos. The <italic>F</italic>1 score was calculated on the eight test videos. (<bold>B</bold>) The best performance of the models trained with input length N=45 and different combos of training and test datasets. (<bold>C</bold>) Model performances with different input lengths.</p></caption><graphic mimetype="image" mime-subtype="tiff" xlink:href="elife-84042-fig3-figsupp1-v2.tif"/></fig><fig id="fig3s2" position="float" specific-use="child-fig"><label>Figure 3—figure supplement 2.</label><caption><title>Error analysis of the best prediction model.</title><p>(<bold>A</bold>) Cartoon showing five types of prediction errors. Red curves indicated scratching probability predicted by the model, and green bars indicated reference scratching trains. (<bold>B</bold>) The incidence rate of each type of errors, calculated by the ratio of the total frames in each type error over the total scratching frames of 8 test videos. Error bar, standard error of the mean (SEM). A real example of false positive prediction (<bold>C1</bold>), the duration of all false positive scratching trains (<bold>C2</bold>), and the frequency distribution of distances between false positive scratching trains to the nearest real scratching train (<bold>C3</bold>). A real example of false negative prediction (<bold>D1</bold>), the duration of all false negative scratching trains (<bold>D2</bold>), and the Type 2 error rate for scratching trains with different duration (<bold>D3</bold>). A real example of blurred boundary prediction (<bold>E1</bold>), the distribution (<bold>E2</bold>), and the average lengths (<bold>E3</bold>) of start and end shift. Error bar, SEM. A real example of missed interval (<bold>F1</bold>), the duration of all missed intervals (<bold>F2</bold>), and Type 4 error rate for intervals with different duration (<bold>F3</bold>). A real example of split scratching train (<bold>G1</bold>), the length of split frames (<bold>G2</bold>), and distribution of the Type 5 error linked to paw licking (<bold>G3</bold>).</p></caption><graphic mimetype="image" mime-subtype="tiff" xlink:href="elife-84042-fig3-figsupp2-v2.tif"/></fig><fig id="fig3s3" position="float" specific-use="child-fig"><label>Figure 3—figure supplement 3.</label><caption><title>Error analysis of the manual quantification.</title><p>(<bold>A</bold>) The incidence rate of each type of errors in manual annotation, calculated by the ratio of the total frames in each type error over the total scratching frames of eight test videos. Error bar, standard error of the mean (SEM). (<bold>B</bold>) The duration of all false positive scratching trains. The duration of all false negative trains (<bold>C1</bold>) and the Type 2 error rate for scratching trains with different duration (<bold>C2</bold>). The distribution (<bold>D1</bold>) and the average lengths (<bold>D2</bold>) of start and end shift. The duration of all missed intervals (<bold>E1</bold>) and the Type 4 error rate for intervals with different duration (<bold>E2</bold>). (<bold>F</bold>) The length of split frames.</p></caption><graphic mimetype="image" mime-subtype="tiff" xlink:href="elife-84042-fig3-figsupp3-v2.tif"/></fig><fig id="fig3s4" position="float" specific-use="child-fig"><label>Figure 3—figure supplement 4.</label><caption><title>Other mouse behaviors were not recognized as scratching behavior.</title><p>Examples of scratching behavior prediction probability during wiping (<bold>A</bold>), grooming (<bold>B</bold>), rearing (<bold>C</bold>), locomotion (<bold>D</bold>), and resting (<bold>E</bold>).</p></caption><graphic mimetype="image" mime-subtype="tiff" xlink:href="elife-84042-fig3-figsupp4-v2.tif"/></fig><fig id="fig3s5" position="float" specific-use="child-fig"><label>Figure 3—figure supplement 5.</label><caption><title>Relationship between prediction errors and the input length or the scratching train duration.</title><p>(<bold>A–C</bold>) The five type error rates of models trained with different input lengths. Error bar, standard error of the mean (SEM). (<bold>D</bold>) The correlation between the scratching train duration and the length of start shift, end shift, or start shift plus end shift. (<bold>E</bold>) The correlation between the average scratching train duration in a video and the prediction accuracy (<italic>F</italic>1 score). (<bold>F</bold>) Frequency distribution of all paw licking duration and those linked to Type 5 error.</p></caption><graphic mimetype="image" mime-subtype="tiff" xlink:href="elife-84042-fig3-figsupp5-v2.tif"/></fig></fig-group></sec><sec id="s2-6"><title>The trained neural network model focused on the scratching hind paw to recognize mouse scratching behavior</title><p>How did the trained neural network model recognize mouse scratching behavior and distinguish them from other behaviors? Although deep learning neural networks are processed as a black box, saliency maps can give some hints (<xref ref-type="bibr" rid="bib32">Selvaraju et al., 2017</xref>), because they plot which parts of each frame (pixels) were mainly used during model prediction. The most salient parts were centered around the scratching hind paw in the scratching frames (<xref ref-type="fig" rid="fig4">Figure 4A, B</xref> and <xref ref-type="fig" rid="fig4s1">Figure 4—figure supplement 1A</xref>), suggesting that the prediction model focused on the features of the scratching hind paw. In some scratching frames, other body parts were also highlighted, such as the two front paws (<xref ref-type="fig" rid="fig4">Figure 4B</xref>), suggesting the model also utilized the positional relationship of these body parts to recognize scratching behavior. In contrast, for other ‘non-scratching’ mouse behaviors, such as wiping, grooming, rearing, and locomotion, the ‘salient’ parts showed no clear association with particular mouse body parts (<xref ref-type="fig" rid="fig4">Figure 4C–F</xref> and <xref ref-type="fig" rid="fig4s1">Figure 4—figure supplement 1B–E</xref>). Together, these saliency maps indicate that the trained neural network learns to focus on the dynamic and static features of scratching for the prediction of mouse scratching behavior.</p><fig-group><fig id="fig4" position="float"><label>Figure 4.</label><caption><title>The prediction model focused on the scratching hind paw for scratching behavior recognition.</title><p>(<bold>A, B</bold>) Saliency map showing the gradient value of each pixel of scratching frames during mouse scratching behavior prediction by the best model. The model focused on the scratching hind paw (<bold>A, B</bold>) and other body parts, such as front paws (<bold>B</bold>).Scale bar, 2 cm. Saliency map showing the gradient value of each pixel of wiping (<bold>C</bold>), grooming (<bold>D</bold>), rearing (<bold>E</bold>), and locomotion (<bold>F</bold>) frames during mouse scratching behavior prediction by the model.</p></caption><graphic mimetype="image" mime-subtype="tiff" xlink:href="elife-84042-fig4-v2.tif"/></fig><fig id="fig4s1" position="float" specific-use="child-fig"><label>Figure 4—figure supplement 1.</label><caption><title>Saliency map of mouse scratching and other behaviors during the prediction.</title><p>Additional saliency maps showing the gradient value of each pixel for frames of scratching (<bold>A</bold>), wiping (<bold>B</bold>), grooming (<bold>C</bold>), rearing (<bold>D</bold>), and locomotion (<bold>E</bold>) during the scratching behavior prediction of the best model.</p></caption><graphic mimetype="image" mime-subtype="tiff" xlink:href="elife-84042-fig4-figsupp1-v2.tif"/></fig></fig-group></sec><sec id="s2-7"><title>Model prediction error analysis</title><p>To further understand the performance of the best trained neural network model, we systematically analyzed its prediction errors in eight test videos (<xref ref-type="fig" rid="fig3s2">Figure 3—figure supplement 2</xref>) and compared to those from the manual quantification (<xref ref-type="fig" rid="fig3s3">Figure 3—figure supplement 3</xref>) and other trained models. We classified the prediction errors into five categories: Type 1, false positive (non-scratching region was predicted as a scratching train); Type 2, false negative (a real scratching train was not recognized); Type 3, blurred boundary (the prediction of the start or end of a scratching train was shifted); Type 4, missed interval (two or more adjacent scratching trains were predicted as one scratching train); and Type 5 split scratching train (one scratching train was predicted as two or more scratching trains) (<xref ref-type="fig" rid="fig3s2">Figure 3—figure supplement 2A</xref>). We found that the dominant prediction error of the trained neural network model was Type 3 error, accounting for around 3% of the total scratching frames, followed by Type 2 and 5 errors accounting for around 1% (<xref ref-type="fig" rid="fig3s2">Figure 3—figure supplement 2B</xref>). For manual quantification, the major errors came from Types 3 and 4, accounting for 10% and 8% of the total scratching frames, respectively (<xref ref-type="fig" rid="fig3s3">Figure 3—figure supplement 3A</xref>).</p><p>For Type 1 error of the model prediction (<xref ref-type="fig" rid="fig3s2">Figure 3—figure supplement 2C1–C3</xref>), the durations of all false positive scratching trains were shorter than 10 frames (0.3 s) (<xref ref-type="fig" rid="fig3s2">Figure 3—figure supplement 2C2</xref>) and temporarily close to a real scratching train (within 30 frames, &lt;1 s) (<xref ref-type="fig" rid="fig3s2">Figure 3—figure supplement 2C3</xref>). They were not caused by confusion with other behaviors, such as wiping, grooming, rearing, locomotion, and resting (<xref ref-type="fig" rid="fig3s4">Figure 3—figure supplement 4</xref>). Type 1 error was also rare for manual annotation (<xref ref-type="fig" rid="fig3s3">Figure 3—figure supplement 3B</xref>).</p><p>The models might miss short scratching trains, hence caused the Type 2 error. Indeed, all missed scratching trains were shorter than 40 frames (&lt;1.3 s) (<xref ref-type="fig" rid="fig3s2">Figure 3—figure supplement 2D1, D2</xref>). For all scratching trains lasting less than 30 frames (&lt;1 s), 18.5% of them were missed by the model prediction. This number decreased to 2.7% for scratching trains spanning between 30 and 60 frames (1–2 s). No scratching train was missed if they were longer than 60 frames (&gt;2 s) (<xref ref-type="fig" rid="fig3s2">Figure 3—figure supplement 2D3</xref>). The Type 2 error positively correlated with the input length (<italic>N</italic>) of prediction models. It became zero or close to zero when models trained with shorter input lengths (3, 13, and 23 frames) (<xref ref-type="fig" rid="fig3s5">Figure 3—figure supplement 5A–C</xref>). For manual annotation, Type 2 error was not common (<xref ref-type="fig" rid="fig3s3">Figure 3—figure supplement 3C1, C2</xref>).</p><p>Type 3 error (<xref ref-type="fig" rid="fig3s2">Figure 3—figure supplement 2E1–E3</xref>) was dominant among all five type errors. The average start and end frame shift of the model prediction were 2.2 and 7.0 frames (<xref ref-type="fig" rid="fig3s2">Figure 3—figure supplement 2E3</xref>), while those of the manual annotation were 11.5 and 12.8 frames (<xref ref-type="fig" rid="fig3s3">Figure 3—figure supplement 3D1, D2</xref>). The start and end frame shift of the manual quantification was similar (~350 ms), which likely reflected the temporal delay of the real time human visual system processing. For the model prediction, which was an off-line frame by frame process, the temporal shifts were less than the human visual processing. In addition, the start of a scratching train was more accurately recognized by the model than the end of a scratching train (<xref ref-type="fig" rid="fig3s2">Figure 3—figure supplement 2E3</xref>). This might reflect the feature of scratching trains. It was relatively clear to determine the start of a scratching train when a mouse lifted its hind paw, but more ambiguous to determine when a mouse put its hind paw back onto the floor to complete a scratching train. The start and end shift did not correlate with the length of a scratching train (<xref ref-type="fig" rid="fig3s5">Figure 3—figure supplement 5D</xref>). Thus, the relative error (percentage of error frames) would decrease when the duration of a scratching train increases. Indeed, the prediction accuracy (as indicated by the <italic>F</italic>1 score) positively correlated (<italic>R</italic><sup>2</sup> = 0.5723) with the average scratching train duration in a video (<xref ref-type="fig" rid="fig3s5">Figure 3—figure supplement 5E</xref>).</p><p>Type 4 error was caused when two adjacent scratching trains were too close to each other and were predicted as one scratching train (<xref ref-type="fig" rid="fig3s2">Figure 3—figure supplement 2F1–F3</xref>). All missed intervals were shorter than 30 frames (&lt;1 s) (<xref ref-type="fig" rid="fig3s2">Figure 3—figure supplement 2F2</xref>). Conversely, 51.4% of intervals less than 30 frames between the two adjacent scratching trains were not recognized. All intervals longer than 30 frames were recognized (<xref ref-type="fig" rid="fig3s2">Figure 3—figure supplement 2F3</xref>). Type 4 error was more common in manual annotation than in the model prediction (<xref ref-type="fig" rid="fig3s3">Figure 3—figure supplement 3E1, E2</xref>).</p><p>Type 5 error occurred when one scratching train was predicted as two or more scratching trains, separated by mispredicted intervals. The average lengths of these mispredicted intervals were around 10 frames by model prediction and around 40 frames by manual annotation (<xref ref-type="fig" rid="fig3s2">Figure 3—figure supplement 2G1, G2</xref> and <xref ref-type="fig" rid="fig3s3">Figure 3—figure supplement 3F</xref>). When reviewing these intervals, we found that more than 80% of them were within or partially overlapped with a paw licking phase (<xref ref-type="fig" rid="fig3s2">Figure 3—figure supplement 2G3</xref>), especially when the duration of the paw licking was more than 30 frames (<xref ref-type="fig" rid="fig3s5">Figure 3—figure supplement 5F</xref>). Thus, it seems likely that the model predicted some long licking frames within a scratching train as ‘non-scratching’. Type 4 and 5 errors reflect the intrinsic complexity of scratching behavior. Human definitions and field consensus, such as using ‘2s’ as the threshold between two adjacent scratching trains (<xref ref-type="bibr" rid="bib8">Darmani and Pandya, 2000</xref>), would help to reduce these types of errors.</p><p>In summary, we have established a novel system combining a customized videotaping box and a well-trained CRNN neural network to automatically identify and quantify mouse scratching behavior with high accuracy. We named it as the Scratch-AID system.</p></sec><sec id="s2-8"><title>Performance of the Scratch-AID system on other major acute itch models</title><p>In addition to the nape, the other commonly used body location to induce mouse itch sensation is the cheek (<xref ref-type="bibr" rid="bib33">Shimada and LaMotte, 2008</xref>). To test whether the Scratch-AID system trained by the nape CQ model could also recognize and quantify scratching behavior of the cheek model, we injected CQ (200 μg in 15 μl saline) into the cheek of five wildtype mice (three males and two females) and recorded seven videos (<xref ref-type="fig" rid="fig5">Figure 5A</xref>). We compared the scratching behavior quantification by the Scratch-AID and manual annotation. The Scratch-AID prediction had 93.4% recall, 94.8% precision, and 0.941 <italic>F</italic>1 score (<xref ref-type="fig" rid="fig5">Figure 5B</xref>), while those of manual quantification were 96.0%, 88.6%, and 0.919 (<xref ref-type="fig" rid="fig5">Figure 5C</xref>). The correlation between the Scratch-AID prediction and reference annotation was 0.9926, and that of the manual quantification was 0.9876 (<xref ref-type="fig" rid="fig5">Figure 5D, E</xref>). The total scratching time in individual videos from both model prediction and manual annotation were close to the reference annotation (<xref ref-type="fig" rid="fig5">Figure 5F</xref>). These results demonstrate that the Scratch-AID system can reliably identify and quantify scratching behavior triggered by acute itch sensation in the cheek.</p><fig id="fig5" position="float"><label>Figure 5.</label><caption><title>The Scratch-AID (Automatic Itch Detection) performance on other acute itch models.</title><p>(<bold>A</bold>) A cartoon showing an acute itch model induced by chloroquine (CQ) injection in the mouse cheek. The average recall, precision, and <italic>F</italic>1 score of Scratch-AID (<bold>B</bold>) or manual annotation (<bold>C</bold>). Error bar, standard error of the mean (SEM). The correlation between model prediction (<bold>D</bold>) or manual quantification (<bold>E</bold>) and reference annotation. <italic>R</italic><sup>2</sup>, Pearson correlation coefficient. (<bold>F</bold>) The comparison among model prediction, manual quantification, and reference annotation. The reference annotation is normalized to 100% shown as the red line. (<bold>G</bold>) A cartoon showing an acute itch model induced by histamine injection in the mouse nape. The average recall, precision and <italic>F</italic>1 score of Scratch-AID (<bold>H</bold>) or manual annotation (<bold>I</bold>). Error bar, SEM. The correlation between model prediction (<bold>J</bold>) or manual quantification (<bold>K</bold>) and reference annotation. <italic>R</italic><sup>2</sup>, Pearson correlation coefficient. (<bold>L</bold>) The comparison among model prediction, manual quantification, and reference annotation. The reference annotation is normalized to 100% shown as the red line.</p></caption><graphic mimetype="image" mime-subtype="tiff" xlink:href="elife-84042-fig5-v2.tif"/></fig><p>Different pruritogens administrated at the same body location trigger scratching behavior with different dynamic features (<xref ref-type="bibr" rid="bib39">Wimalasena et al., 2021</xref>). Thus, we tested whether the Scratch-AID system trained by CQ injection could recognize scratching behavior triggered by a different pruritogen, histamine. 100 μg histamine (in 15 μl saline) was intradermally injected into the nape and four videos were recorded (<xref ref-type="fig" rid="fig5">Figure 5G</xref>). The recall, precision, and <italic>F</italic>1 score from the Scratch-AID prediction were 96.6%, 90.91%, and 0.936 (<xref ref-type="fig" rid="fig5">Figure 5H</xref>), while those of the manual annotation were 96.3%, 80.5%, and 0.877 (<xref ref-type="fig" rid="fig5">Figure 5I</xref>). The correlation between the Scratch-AID prediction and reference annotation is 0.9707, and that of the manual quantification is 0.9895 (<xref ref-type="fig" rid="fig5">Figure 5J, K</xref>). The total scratching time in individual videos from both model prediction and manual annotation was similar to those of reference annotation (<xref ref-type="fig" rid="fig5">Figure 5L</xref>). Taken together, these data show that our Scratch-AID system, although only trained with the CQ nape acute itch model, can recognize and quantify scratching behavior of acute itch models induced at different skin locations or triggered by different pruritogens, and that the prediction accuracy of Scratch-AID is comparable to that of the manual annotation.</p></sec><sec id="s2-9"><title>Performance of the Scratch-AID system on a chronic itch model</title><p>Chronic itch is a devastating symptom and severely affects the quality of patients’ life (<xref ref-type="bibr" rid="bib40">Yosipovitch and Fleischer, 2003</xref>; <xref ref-type="bibr" rid="bib42">Yu et al., 2021</xref>). Investigating the underlying mechanisms using the mouse model is important for developing novel therapies for treating chronic itch in various conditions. To test whether the Scratch-AID system could be used to study mouse chronic itch models, we generated a squaric acid dibutylester (SADBE) induced contact dermatitis chronic itch model (<xref ref-type="bibr" rid="bib2">Beattie et al., 2022</xref>; <xref ref-type="bibr" rid="bib30">Qu et al., 2015</xref>) and recorded nine videos from three wildtype mice (<xref ref-type="fig" rid="fig6">Figure 6A</xref>). Affected mice displayed spontaneous scratching toward the nape and/or head. Noticeably, dynamic features of spontaneous scratching behavior under this chronic itch condition were different from those exhibited by the CQ acute itch model: the total scratching time was less for the same given period of time (20 min) (<xref ref-type="fig" rid="fig6s1">Figure 6—figure supplement 1A</xref>), and the average duration of the scratching trains was shorter (53 frames on average) than the acute scratching behavior induced by CQ (280 frames on average) (<xref ref-type="fig" rid="fig6s1">Figure 6—figure supplement 1B</xref> and <xref ref-type="fig" rid="fig1">Figure 1F</xref>). Despite these considerable differences, the recall, precision, and <italic>F</italic>1 score of the Scratch-AID prediction were 84.1%, 88.5%, and 0.862, respectively, compared to 84.1%, 66.3%, and 0.740 of manual annotation (<xref ref-type="fig" rid="fig6">Figure 6B, C</xref>). A likely reason for the decreased recall and precision of both model prediction and manual annotation was the dominant short scratching trains in this chronic itch model (<xref ref-type="fig" rid="fig6s1">Figure 6—figure supplement 1B</xref>). The correlation between the model prediction and reference annotation was 0.9845 (<xref ref-type="fig" rid="fig6">Figure 6D</xref>), which was comparable to the manual annotation 0.9887 (<xref ref-type="fig" rid="fig6">Figure 6E</xref>). The total scratching time quantified by the model was slightly more accurate than the manual annotation (<xref ref-type="fig" rid="fig6">Figure 6F</xref>). From the prediction traces (an example shown in <xref ref-type="fig" rid="fig6">Figure 6G</xref>), the Scratch-AID system was capable to capture the low-frequency short scratching trains.</p><fig-group><fig id="fig6" position="float"><label>Figure 6.</label><caption><title>The Scratch-AID performance on a chronic itch model.</title><p>(<bold>A</bold>) A cartoon showing a squaric acid dibutylester (SADBE) induced chronic itch model. The average recall, precision, and <italic>F</italic>1 score of Scratch-AID (<bold>B</bold>) or manual annotation (<bold>C</bold>). Error bar, standard error of the mean (SEM). The correlation between model prediction (<bold>D</bold>) or manual quantification (<bold>E</bold>) and reference annotation. <italic>R</italic><sup>2</sup>, Pearson correlation coefficient. (<bold>F</bold>) The comparison among model prediction, manual quantification, and reference annotation. The reference annotation is normalized to 100% shown as the red line. (<bold>G</bold>) An example scratching probability trace (red curve) predicted by the model and aligned with the reference annotation (green bar) (left). Zoom-in (right panel) of the blue square part showing nice alignment of the model prediction with the reference annotation.</p></caption><graphic mimetype="image" mime-subtype="tiff" xlink:href="elife-84042-fig6-v2.tif"/></fig><fig id="fig6s1" position="float" specific-use="child-fig"><label>Figure 6—figure supplement 1.</label><caption><title>Different dynamic features of chronic and acute itch models.</title><p>(<bold>A</bold>) The percentage of scratching and non-scratching frames in the squaric acid dibutylester (SADBE) chronic itch model (<italic>n</italic> = 9 videos) and chloroquine (CQ) nape acute itch model (<italic>n</italic> = 40 videos). (<bold>B</bold>) Frequency distribution of scratching train duration of SADBE chronic itch model and CQ nape acute itch model.</p></caption><graphic mimetype="image" mime-subtype="tiff" xlink:href="elife-84042-fig6-figsupp1-v2.tif"/></fig></fig-group></sec><sec id="s2-10"><title>Application of the Scratch-AID system in anti-itch drug screening</title><p>Finally, we tested whether the Scratch-AID system could be applied for anti-itch drug screening. Here, we used the histaminergic acute itch nape model and the pretreatment of Benadryl (<xref ref-type="bibr" rid="bib22">Loew et al., 1946</xref>), an FDA approved anti-histaminergic itch cream, as a proof-of-principle example. Benadryl or control cream was topically applied onto the mouse nape skin 1 hr before the intradermal injection of histamine (200 μg in 15 μl saline). Scratching behavior was then recorded for 20 min from six C57 wildtype mice (two males, four females) in the Benardryl treated group and seven C57 wildtype mice (two males, five females) in the control group (<xref ref-type="fig" rid="fig7">Figure 7A</xref>). Quantification of the total scratching time (frames) using the Scratch-AID showed a significant reduction with Benardryl treatment (<xref ref-type="fig" rid="fig7">Figure 7B</xref>). Similar results were found by manual annotation (<xref ref-type="fig" rid="fig7">Figure 7C</xref>). These results suggest that the Scratch-AID system is sensitive to detect the change of scratching behavior after an anti-itch drug treatment, which highlights the potential use of the Scratch-AID system in high-throughput and large-scale anti-itch drug screenings.</p><fig id="fig7" position="float"><label>Figure 7.</label><caption><title>Application of the Scratch-AID (Automatic Itch Detection) system in a drug screening paradigm.</title><p>(<bold>A</bold>) A diagram showing the experimental design of an anti-itch drug test. Quantification of scratching behavior in anti-itch cream treated group or control group by Scratch-AID (<bold>B</bold>) or manual annotation (<bold>C</bold>). Error bar, standard error of the mean (SEM). Differences between the two groups were analyzed using unpaired two-tailed Student’s <italic>t</italic>-test, ** p &lt; 0.01.</p></caption><graphic mimetype="image" mime-subtype="tiff" xlink:href="elife-84042-fig7-v2.tif"/></fig></sec></sec><sec id="s3" sec-type="discussion"><title>Discussion</title><p>Scratching is an itch-specific behavior, and the mouse is the major model animal to study itch mechanisms and to develop novel anti-itch drugs. In this study, we developed a new system, Scratch-AID, which combined a customized videotaping box and a well-trained neural network, for automatic quantification of mouse scratching behavior with high accuracy. Its performance is comparable to the manual annotation on major itch models and an anti-itch drug screening paradigm. As far as we are aware, this is the first deep learning-based system that can achieve such high accuracy and broad generalization.</p><p>It is remarkable that a model trained with CQ induced acute nape itch videos could reliably recognize scratching behavior in other itch models, even when the body sites (nape vs. cheek) or the dynamics of scratching behavior (e.g., acute vs. chronic itch, different pruritogens) were different. The impressive performance and generalization of the Scratch-AID system are likely attributed to the high-quality and reproducible video recording; a large amount of high-quality training datasets with the frame-by-frame annotation; efficient CRNN deep learning neural network design; and training parameter optimizations.</p><p>Variable video recording condition is a major barrier that has prevented the adoption of a trained neural network by different laboratories. The trained models usually do not perform well with videos recorded under different conditions (illumination, field size, image resolution, magnification of mice, image angle, backgrounds, etc.). Thus, we built a customized videotaping box to provide a reproducible and high-quality recording environment (<xref ref-type="fig" rid="fig1">Figure 1B</xref>). This helps to standardize the videotaping process, reduce the noise, and facilitate the stable performance and generalization of the trained deep learning models. Meanwhile, this video recording box is easy to be set up, scaled up, and adopted. Combined with the well-trained prediction model that has a comparable performance as manual quantification, the Scratch-AID system is ready to replace the manual quantification.</p><p>The amount and quality of training datasets are important to train a high accurate and generalized model. In general, the performance of a prediction model positively corelates with the size of the training dataset. In this study, we recorded 40 scratching videos in total and chose the CQ induced acute nape itch model, which triggered robust scratching behavior in mice, for model training and testing. The large number of scratching videos and the high amount of incidence of targeted behavior in each video provided a high-quality training dataset. In addition, the clear definition of scratching train and accurate annotation of videos, as conducted frame by frame, were also critical for training, testing, and error analysis.</p><p>CRNN is the classical deep learning neural network for analysis of animal behaviors. Here, we used ResNet (<xref ref-type="bibr" rid="bib15">He et al., 2016</xref>) for the CNN part, which simplifies the neural network model by constructing a residual learning block through a shortcut connection of identity mapping (<xref ref-type="fig" rid="fig2s1">Figure 2—figure supplement 1B</xref>). GRU was used for the RNN part for extracting dynamic features (<xref ref-type="fig" rid="fig2s1">Figure 2—figure supplement 1C</xref>). Our study shows that ResNet and GRU make a good combination of deep learning architecture for analyzing animal behaviors. In addition, our network design is highly efficient that the accuracy plateau was achieved after only 10 epochs of training.</p><p>There is still some room to improve the performance of our predication models. To increase the capability of Scratch-AID to capture short scratching trains, we could train the CRNN neural network with scratching videos from chronic itch models. In addition, optimization of training parameters could also help with the improvement of prediction accuracy. For example, different error types varied when changing the input length (<xref ref-type="fig" rid="fig3s2">Figure 3—figure supplement 2B</xref> and <xref ref-type="fig" rid="fig3s5">Figure 3—figure supplement 5A–C</xref>). The Type 2 and 4 errors increased when the input length increased, while Type 1, 3, and 5 errors decreased when the input length increased. Thus, it is a trade-off to optimize and select the best input length for different itch models. Increasing the size of the training dataset may also help to get better prediction models. When checking the videos with relatively low prediction accuracy, for example, the video number 5 (V5) in the chronic itch model (<xref ref-type="fig" rid="fig6">Figure 6F</xref>), we found that the missed scratching frames could be due to a rare posture during scratching behavior, in which the scratching hind paw was partially occluded by the tail (<xref ref-type="video" rid="video3">Video 3</xref>). Thus, with more training videos containing some rare scratching postures, the trained neural network models could be more ‘knowledgeable’ for the diversity of scratching behavior.</p><media mimetype="video" mime-subtype="mp4" xlink:href="elife-84042-video3.mp4" id="video3"><label>Video 3.</label><caption><title>An example of a rare posture during the scratching behavior with scratching hind paw partially occluded by the tail.</title></caption></media><p>Quantification of animal behaviors is critical for understanding the underlying molecular, cellular, and circuit mechanisms. Compared to manual analysis, the deep learning-based automatic analysis will not only improve efficiency and accuracy, but also reduce human bias and errors. Along this direction, we developed the Scratch-AID system to achieve automatic quantification of mouse scratching behavior. Our study also provides useful insights for developing new deep learning neural network models to achieve automatic analysis of other animal behaviors.</p></sec><sec id="s4" sec-type="materials|methods"><title>Materials and methods</title><sec id="s4-1"><title>Mouse lines and treatments</title><p>Mice (C57BL/6J purchased from The Jackson Laboratory, RRID: <ext-link ext-link-type="uri" xlink:href="https://identifiers.org/RRID/RRID:IMSR_JAX:000664">IMSR_JAX:000664</ext-link>) were housed in the John Morgan animal facility at the University of Pennsylvania. All animal treatments were conducted in accordance with protocols approved by the Institutional Animal Care and Use Committee and the guidelines of the National Institutes of Health (Protocol No. 804886).</p></sec><sec id="s4-2"><title>Acute itch model</title><p>Acute itch models were conducted as previously described (<xref ref-type="bibr" rid="bib7">Cui et al., 2022</xref>). Mouse cheeks or napes were shaved 3 days prior to experiments, and mice were placed in the videotaping box for acclimation (15 min/day for 3 days). At the day of experiment, mice were first acclimated to the videotaping box for 5  min, CQ (Sigma, C6628) (200 µg in 15 μl saline) or histamine (Sigma, H7250) (100 µg in 15 μl saline) was then intradermally injected into either the cheek or the nape, and scratching behavior was recorded for 20 min.</p></sec><sec id="s4-3"><title>Behavior recording</title><p>The mouse scratching behavior was recorded using a Microsoft laptop (Windows 10 Pro, purchased from Amazon) connecting to a web camera (Logitech C920e Business Webcam, purchased from Amazon). Logitech Capture 2.06.12 Software was used to adjust the following recording parameters: brightness 170, contrast 0, resolution 720 × 720, frame rate 30 fps. The brightness and contrast could be adjusted according to ambient light to achieve consistent illumination.</p></sec><sec id="s4-4"><title>Annotation of mouse scratching behavior</title><p>The start of a scratching train was defined as when the mouse started to lift up its hind paw and prepared to scratch at the beginning of a scratching train. For the end of a scratching train, there were two cases. If the mouse did not lick its hind paw after last scratching bout, the end frame would be when the mouse put its hind paw back onto the ground; if the mouse licked its hind paw after last scratching bout, the end frame would be when the mouse put its hind paw into the mouth. For the manual annotation, each video was manually watched at the normal (1×) speed (30 frames/s). The start and end time point of each scratching train was manually annotated, the time point was converted into the frame number (30 frames/s) for downstream analysis. For the reference annotation, each video was first converted into individual frames using python package OpenCV (<xref ref-type="bibr" rid="bib5">Bradski, 2000</xref>). Then the start and end of each scratching train was determined frame by frame. Frames within a scratching train were defined as ‘scratching’ frames, otherwise defined as ‘non-scratching’ frames. For CQ cheek acute itch video number 2 (<xref ref-type="fig" rid="fig5">Figure 5F</xref> V2) and SADBE chronic itch video number 5 (<xref ref-type="fig" rid="fig6">Figure 6F</xref> V5), long lickings (&gt;60 frames) during scratching trains were annotated as non-scratching frames for the model evaluation. For group comparison, the annotation was performed in a double-blind manner.</p></sec><sec id="s4-5"><title>Training sample preparation and preprocessing</title><p>In the training procedure, 40 videos from 10 mice were first randomly split into 32 training videos from 8 mice (V1–V32), and 8 test videos from 2 mice (V33–V40). For cross-validation, training and test videos were rotated. Test videos were V1–V8, V9–V16, V17–V24, V25–V32, or V33–V40, and the rest of videos were used as training videos. For the model training, input was prepared by the following procedure: First, each video was converted into frames; then, consecutive <italic>N</italic> frames (<italic>N</italic> = 3, 5, 7, 13, 23, or 45) were selected as one input with an interval of 4–10 frames between two adjacent inputs. Then each input was annotated as ‘scratching’ (class 1) if more than <italic>N</italic>/2 frames were scratching frames, otherwise labeled as ‘non-scratching’ (class 0).</p><p>All frames in each input were first converted into gray scale images and resized to 300 × 300. Then a random square crop with size ranging from 288 × 288 to 300 × 300 was applied and followed by a random horizontal and vertical flip with probability of 0.5. Finally, these frames were resized to 256 × 256 and fed into the CRNN network.</p></sec><sec id="s4-6"><title>CRNN architecture</title><p>Our CRNN model (<xref ref-type="fig" rid="fig2">Figure 2A</xref> and <xref ref-type="fig" rid="fig2s1">Figure 2—figure supplement 1</xref>) consisted of a CNN part, an RNN part, and a full connection (FC) part. The CNN was modified from ResNet-18 (<xref ref-type="bibr" rid="bib15">He et al., 2016</xref> ) by changing the final FC layer into an FC layer with embedding size 256. The RNN consisted of two bidirectional GRU layers with hidden vector size 512. The FC part consisted of two FC layers with embedding size 256 and ReLu activation, and embedding size 2 for the final prediction. The prediction results were transformed into maximum value in the final output vector.</p></sec><sec id="s4-7"><title>Model training and predication</title><p>The model was trained by PyTorch (version 1.10.2, RRID:<ext-link ext-link-type="uri" xlink:href="https://identifiers.org/RRID/RRID:SCR_018536">SCR_018536</ext-link>) (<xref ref-type="bibr" rid="bib28">Paszke et al., 2019</xref>). For model training hyperparameters, batch size was set as 16 or 32 depending on the input size. The max epoch was set as 20. ADAM optimizer was used with initial learning rate 10<sup>−4</sup> and the learning rate reduced by multiplying factor 0.3 every 5 epochs. The binary cross entropy was used as loss function. Dropout rate of FC layer was set as 0.2. The model was trained on a customized desktop with Intel i9-10900k CPU (purchased from Newegg), 64 GB RAM (CORSAIR Vengeance LPX 64 GB, purchased from Newegg), and NVIDIA GeForce RTX 3090 with 24 GB memory (purchased from Amazon).</p><p>For model prediction on new videos, the input preparation was similar to the training dataset, except that the adjacent inputs were only 1 frame apart. Each individual frame was predicted as ‘scratching’ or ‘non-scratching’ based on the following rule: the prediction of the middle frame of each input would be the same as the input prediction. For the few frames at the beginning or at the end of each video (depending on the input length <italic>N</italic>) that could not be the middle frame of an input, they were predicted as ‘scratching’ or ‘non-scratching’ based on the first input or last input prediction.</p></sec><sec id="s4-8"><title>Saliency map</title><p>To obtain the saliency map, we first calculated gradient value of each pixel and kept the only positive gradients, then rescaled into range 0–1 based on a previous published paper (<xref ref-type="bibr" rid="bib32">Selvaraju et al., 2017</xref>). Then we generated the heatmap based on the gradient values (&lt;0.1 transparent, 0.1–0.6 light blue to dark blue, &gt;0.6 dark blue) and stacked the heatmap onto the original frame.</p></sec><sec id="s4-9"><title>SADBE chronic itch model</title><p>Mice (8–12 weeks) were singly housed before starting the behavior experiment. In day 1, mice were individually anesthetized in chamber using isoflurane until they did not move and showed decreased respiration rates. Mice were continued for anesthesia with a nose cone to allow access to the body of the mouse. After ensuring a mouse was fully anesthetized, the abdominal skin was shaved. In day 4, after anesthesia similar in day 1, 25 µl 1% SADBE in acetone (Sigma, 3339792) was applied to the shaved area of the abdominal skin. After application of SADBE, anesthesia continued for 3 more minutes to ensure SADBE fully absorbed before putting then back in the home cage. In days 9–12, mice were habituated in the videotaping box for 1 hr and 15 min. In day 11, the nape was shaved after habituation. In days 13–18, mice were habituated in the videotaping box for 1 hr and 15 min, then 25 µl of 1% SADBE was applied to the nape. In day 18, after 1% SADBE application, each mouse was put back into the videotaping box and recorded for three videos.</p></sec><sec id="s4-10"><title>Anti-itch drug experiment</title><p>The nape of mice (8–12 weeks, two males, five females) was shaved 3 days prior to experiments. Mice were placed in the videotaping box for acclimation (15 min per day for 3 days). At the day of experiment, mice were randomly selected and acclimated in the videotaping box for 5 min. Then, 0.2 g anti-itch cream (Benadryl, Johnson &amp; Johnson Consumer Inc, purchased from CVS) or control cream (Neutrogena, Johnson Consumer Inc, purchased from CVS) was applied into the nape. One hour later, histamine (200 µg in 15 µl saline) was intradermally injected into the nape, and scratching behavior was recorded for 20 min. The experimenter who did the histamine injection was blind to anti-itch cream or control cream.</p></sec><sec id="s4-11"><title>Illustration drawing</title><p>Cartoons with mice were made partially in BioRender (<xref ref-type="bibr" rid="bib3">BioRender, 2022</xref>, RRID:<ext-link ext-link-type="uri" xlink:href="https://identifiers.org/RRID/RRID:SCR_018361">SCR_018361</ext-link>).</p></sec><sec id="s4-12"><title>Statistical analysis</title><p>Data and statistical analyses were performed using Prism 6.0 (GraphPad Software, RRID:<ext-link ext-link-type="uri" xlink:href="https://identifiers.org/RRID/RRID:SCR_002798">SCR_002798</ext-link>). The criteria for significance were: ns (not significant) p ≥ 0.05, *p &lt; 0.05, **p &lt; 0.01, ***p &lt; 0.001. Differences in means between two groups were analyzed using unpaired two-tailed Student’s <italic>t</italic>-test.</p></sec></sec></body><back><sec sec-type="additional-information" id="s5"><title>Additional information</title><fn-group content-type="competing-interest"><title>Competing interests</title><fn fn-type="COI-statement" id="conf1"><p>No competing interests declared</p></fn></fn-group><fn-group content-type="author-contribution"><title>Author contributions</title><fn fn-type="con" id="con1"><p>Conceptualization, Resources, Data curation, Software, Formal analysis, Supervision, Validation, Investigation, Visualization, Methodology, Writing - original draft, Project administration, Writing - review and editing</p></fn><fn fn-type="con" id="con2"><p>Conceptualization, Resources, Data curation, Software, Formal analysis, Supervision, Validation, Investigation, Visualization, Methodology, Writing - original draft, Project administration, Writing - review and editing</p></fn><fn fn-type="con" id="con3"><p>Conceptualization, Resources, Data curation, Software, Formal analysis, Validation, Investigation, Visualization, Methodology, Writing - review and editing</p></fn><fn fn-type="con" id="con4"><p>Conceptualization, Data curation, Software, Formal analysis, Investigation, Visualization, Methodology, Writing - review and editing</p></fn><fn fn-type="con" id="con5"><p>Conceptualization, Data curation, Visualization, Methodology, Writing - review and editing</p></fn><fn fn-type="con" id="con6"><p>Conceptualization, Data curation, Methodology</p></fn><fn fn-type="con" id="con7"><p>Conceptualization, Software, Methodology</p></fn><fn fn-type="con" id="con8"><p>Conceptualization, Data curation, Software, Methodology</p></fn><fn fn-type="con" id="con9"><p>Data curation, Software, Methodology</p></fn><fn fn-type="con" id="con10"><p>Data curation, Visualization, Methodology</p></fn><fn fn-type="con" id="con11"><p>Data curation, Visualization</p></fn><fn fn-type="con" id="con12"><p>Data curation</p></fn><fn fn-type="con" id="con13"><p>Data curation</p></fn><fn fn-type="con" id="con14"><p>Data curation</p></fn><fn fn-type="con" id="con15"><p>Data curation</p></fn><fn fn-type="con" id="con16"><p>Data curation</p></fn><fn fn-type="con" id="con17"><p>Data curation</p></fn><fn fn-type="con" id="con18"><p>Data curation</p></fn><fn fn-type="con" id="con19"><p>Data curation</p></fn><fn fn-type="con" id="con20"><p>Data curation</p></fn><fn fn-type="con" id="con21"><p>Conceptualization, Data curation, Methodology, Writing - review and editing</p></fn><fn fn-type="con" id="con22"><p>Conceptualization, Supervision, Methodology, Writing - review and editing</p></fn><fn fn-type="con" id="con23"><p>Conceptualization, Supervision, Funding acquisition, Methodology, Writing - review and editing</p></fn><fn fn-type="con" id="con24"><p>Conceptualization, Resources, Data curation, Software, Formal analysis, Supervision, Funding acquisition, Validation, Investigation, Visualization, Methodology, Project administration, Writing - review and editing</p></fn></fn-group><fn-group content-type="ethics-information"><title>Ethics</title><fn fn-type="other"><p>Mice were housed in the John Morgan animal facility at the University of Pennsylvania. All animal treatments were conducted in accordance with protocols approved by the Institutional Animal Care and Use Committee and the guidelines of the National Institutes of Health (Protocol No. 804886).</p></fn></fn-group></sec><sec sec-type="supplementary-material" id="s6"><title>Additional files</title><supplementary-material id="transrepform"><label>Transparent reporting form</label><media xlink:href="elife-84042-transrepform1-v2.pdf" mimetype="application" mime-subtype="pdf"/></supplementary-material><supplementary-material id="supp1"><label>Supplementary file 1.</label><caption><title>Mouse information used in the recording of the training and test videos.</title></caption><media xlink:href="elife-84042-supp1-v2.docx" mimetype="application" mime-subtype="docx"/></supplementary-material><supplementary-material id="supp2"><label>Supplementary file 2.</label><caption><title>Scratching behavior summary in the 40 training and test videos.</title></caption><media xlink:href="elife-84042-supp2-v2.docx" mimetype="application" mime-subtype="docx"/></supplementary-material></sec><sec sec-type="data-availability" id="s7"><title>Data availability</title><p>The training and test videos generated during the current study can be downloaded from DRYAD (<ext-link ext-link-type="uri" xlink:href="https://doi.org/10.5061/dryad.mw6m9060s">https://doi.org/10.5061/dryad.mw6m9060s</ext-link>). The codes for model training and test can be downloaded from GitHub (<ext-link ext-link-type="uri" xlink:href="https://github.com/taimeimiaole/Scratch-AID">https://github.com/taimeimiaole/Scratch-AID</ext-link>, copy archived at <ext-link ext-link-type="uri" xlink:href="https://archive.softwareheritage.org/swh:1:dir:9a8b57fd061692f85b814e0b064fc8cefeec1839;origin=https://github.com/taimeimiaole/Scratch-AID;visit=swh:1:snp:801227cfb2c4f3f5609131dba4f4e22101b85f73;anchor=swh:1:rev:d8a1e6b94e54be2c857285d74623e495a6bd47bf">swh:1:rev:d8a1e6b94e54be2c857285d74623e495a6bd47bf</ext-link>).</p><p>The following dataset was generated:</p><p><element-citation publication-type="data" specific-use="isSupplementedBy" id="dataset1"><person-group person-group-type="author"><name><surname>Luo</surname><given-names>W</given-names></name></person-group><year iso-8601-date="2022">2022</year><data-title>Data From: Scratch-AID: a deep-learning based system for automatic detection of mouse scratching behavior with high accuracy</data-title><source>Dryad Digital Repository</source><pub-id pub-id-type="doi">10.5061/dryad.mw6m9060s</pub-id></element-citation></p></sec><ack id="ack"><title>Acknowledgements</title><p>We thank the Luo and Ma lab members for their help and support. We thank Dr. Ji Zhu, Mr. Simin Liu, Dr. You Lv, Ms. Yakun Wang, and Dr. Wei Yang for their help and suggestions for this project. This work was supported by NSF grant (DMS-1854770) of Dr. Arsuaga, NIH R01 (NS083702) of Dr. Luo, and R34 (NS118411) of Drs. Ding and Luo.</p></ack><ref-list><title>References</title><ref id="bib1"><element-citation publication-type="confproc"><person-group person-group-type="author"><name><surname>Abadi</surname><given-names>M</given-names></name><name><surname>Barham</surname><given-names>P</given-names></name><name><surname>Chen</surname><given-names>J</given-names></name><name><surname>Chen</surname><given-names>Z</given-names></name><name><surname>Davis</surname><given-names>A</given-names></name><name><surname>Dean</surname><given-names>J</given-names></name><name><surname>Devin</surname><given-names>M</given-names></name><name><surname>Ghemawat</surname><given-names>S</given-names></name><name><surname>Irving</surname><given-names>G</given-names></name><name><surname>Isard</surname><given-names>M</given-names></name></person-group><year iso-8601-date="2016">2016</year><article-title>TensorFlow}: a system for {Large-Scale} machine learning</article-title><conf-name>12th USENIX symposium on operating systems design and implementation (OSDI)</conf-name></element-citation></ref><ref id="bib2"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Beattie</surname><given-names>K</given-names></name><name><surname>Jiang</surname><given-names>H</given-names></name><name><surname>Gautam</surname><given-names>M</given-names></name><name><surname>MacVittie</surname><given-names>MK</given-names></name><name><surname>Miller</surname><given-names>B</given-names></name><name><surname>Ma</surname><given-names>M</given-names></name><name><surname>Liu</surname><given-names>Q</given-names></name><name><surname>Luo</surname><given-names>W</given-names></name></person-group><year iso-8601-date="2022">2022</year><article-title>Trpc3 antagonizes pruritus in a mouse contact dermatitis model</article-title><source>The Journal of Investigative Dermatology</source><volume>142</volume><fpage>1136</fpage><lpage>1144</lpage><pub-id pub-id-type="doi">10.1016/j.jid.2021.08.433</pub-id><pub-id pub-id-type="pmid">34570999</pub-id></element-citation></ref><ref id="bib3"><element-citation publication-type="software"><person-group person-group-type="author"><collab>BioRender</collab></person-group><year iso-8601-date="2022">2022</year><data-title>BioRender</data-title><source>BioRender.Com</source></element-citation></ref><ref id="bib4"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Bohnslav</surname><given-names>JP</given-names></name><name><surname>Wimalasena</surname><given-names>NK</given-names></name><name><surname>Clausing</surname><given-names>KJ</given-names></name><name><surname>Dai</surname><given-names>YY</given-names></name><name><surname>Yarmolinsky</surname><given-names>DA</given-names></name><name><surname>Cruz</surname><given-names>T</given-names></name><name><surname>Kashlan</surname><given-names>AD</given-names></name><name><surname>Chiappe</surname><given-names>ME</given-names></name><name><surname>Orefice</surname><given-names>LL</given-names></name><name><surname>Woolf</surname><given-names>CJ</given-names></name><name><surname>Harvey</surname><given-names>CD</given-names></name></person-group><year iso-8601-date="2021">2021</year><article-title>DeepEthogram, a machine learning pipeline for supervised behavior classification from raw pixels</article-title><source>eLife</source><volume>10</volume><elocation-id>e63377</elocation-id><pub-id pub-id-type="doi">10.7554/eLife.63377</pub-id><pub-id pub-id-type="pmid">34473051</pub-id></element-citation></ref><ref id="bib5"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Bradski</surname><given-names>G</given-names></name></person-group><year iso-8601-date="2000">2000</year><article-title>The opencv library</article-title><source>Dr Dobb’s Journal: Software Tools for the Professional Programmer</source><volume>25</volume><fpage>120</fpage><lpage>123</lpage></element-citation></ref><ref id="bib6"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Cevikbas</surname><given-names>F</given-names></name><name><surname>Lerner</surname><given-names>EA</given-names></name></person-group><year iso-8601-date="2020">2020</year><article-title>Physiology and pathophysiology of itch</article-title><source>Physiological Reviews</source><volume>100</volume><fpage>945</fpage><lpage>982</lpage><pub-id pub-id-type="doi">10.1152/physrev.00017.2019</pub-id><pub-id pub-id-type="pmid">31869278</pub-id></element-citation></ref><ref id="bib7"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Cui</surname><given-names>L</given-names></name><name><surname>Guo</surname><given-names>J</given-names></name><name><surname>Cranfill</surname><given-names>SL</given-names></name><name><surname>Gautam</surname><given-names>M</given-names></name><name><surname>Bhattarai</surname><given-names>J</given-names></name><name><surname>Olson</surname><given-names>W</given-names></name><name><surname>Beattie</surname><given-names>K</given-names></name><name><surname>Challis</surname><given-names>RC</given-names></name><name><surname>Wu</surname><given-names>Q</given-names></name><name><surname>Song</surname><given-names>X</given-names></name><name><surname>Raabe</surname><given-names>T</given-names></name><name><surname>Gradinaru</surname><given-names>V</given-names></name><name><surname>Ma</surname><given-names>M</given-names></name><name><surname>Liu</surname><given-names>Q</given-names></name><name><surname>Luo</surname><given-names>W</given-names></name></person-group><year iso-8601-date="2022">2022</year><article-title>Glutamate in primary afferents is required for itch transmission</article-title><source>Neuron</source><volume>110</volume><fpage>809</fpage><lpage>823</lpage><pub-id pub-id-type="doi">10.1016/j.neuron.2021.12.007</pub-id><pub-id pub-id-type="pmid">34986325</pub-id></element-citation></ref><ref id="bib8"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Darmani</surname><given-names>NA</given-names></name><name><surname>Pandya</surname><given-names>DK</given-names></name></person-group><year iso-8601-date="2000">2000</year><article-title>Involvement of other neurotransmitters in behaviors induced by the cannabinoid CB1 receptor antagonist SR 141716A in naive mice</article-title><source>Journal of Neural Transmission</source><volume>107</volume><fpage>931</fpage><lpage>945</lpage><pub-id pub-id-type="doi">10.1007/s007020070043</pub-id><pub-id pub-id-type="pmid">11041273</pub-id></element-citation></ref><ref id="bib9"><element-citation publication-type="confproc"><person-group person-group-type="author"><name><surname>Dey</surname><given-names>R</given-names></name><name><surname>Salem</surname><given-names>FM</given-names></name></person-group><year iso-8601-date="2017">2017</year><article-title>Gate-variants of Gated Recurrent Unit (GRU) neural networks</article-title><conf-name>2017 IEEE 60th International Midwest Symposium on Circuits and Systems (MWSCAS</conf-name><pub-id pub-id-type="doi">10.1109/MWSCAS.2017.8053243</pub-id></element-citation></ref><ref id="bib10"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Elliott</surname><given-names>P</given-names></name><name><surname>G’Sell</surname><given-names>M</given-names></name><name><surname>Snyder</surname><given-names>LM</given-names></name><name><surname>Ross</surname><given-names>SE</given-names></name><name><surname>Ventura</surname><given-names>V</given-names></name></person-group><year iso-8601-date="2017">2017</year><article-title>Automated acoustic detection of mouse scratching</article-title><source>PLOS ONE</source><volume>12</volume><elocation-id>e0179662</elocation-id><pub-id pub-id-type="doi">10.1371/journal.pone.0179662</pub-id><pub-id pub-id-type="pmid">28678797</pub-id></element-citation></ref><ref id="bib11"><element-citation publication-type="preprint"><person-group person-group-type="author"><name><surname>Graves</surname><given-names>A</given-names></name></person-group><year iso-8601-date="2013">2013</year><article-title>Generating Sequences with Recurrent Neural Networks</article-title><source>arXiv</source><ext-link ext-link-type="uri" xlink:href="https://arxiv.org/abs/1308.0850">https://arxiv.org/abs/1308.0850</ext-link></element-citation></ref><ref id="bib12"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Gu</surname><given-names>J</given-names></name><name><surname>Wang</surname><given-names>Z</given-names></name><name><surname>Kuen</surname><given-names>J</given-names></name><name><surname>Ma</surname><given-names>L</given-names></name><name><surname>Shahroudy</surname><given-names>A</given-names></name><name><surname>Shuai</surname><given-names>B</given-names></name><name><surname>Liu</surname><given-names>T</given-names></name><name><surname>Wang</surname><given-names>X</given-names></name><name><surname>Wang</surname><given-names>G</given-names></name><name><surname>Cai</surname><given-names>J</given-names></name><name><surname>Chen</surname><given-names>T</given-names></name></person-group><year iso-8601-date="2018">2018</year><article-title>Recent advances in convolutional neural networks</article-title><source>Pattern Recognition</source><volume>77</volume><fpage>354</fpage><lpage>377</lpage><pub-id pub-id-type="doi">10.1016/j.patcog.2017.10.013</pub-id></element-citation></ref><ref id="bib13"><element-citation publication-type="book"><person-group person-group-type="author"><name><surname>Gulli</surname><given-names>A</given-names></name><name><surname>Pal</surname><given-names>S</given-names></name></person-group><year iso-8601-date="2017">2017</year><source>Deep Learning with Keras</source><publisher-name>Packt Publishing Ltd</publisher-name></element-citation></ref><ref id="bib14"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Han</surname><given-names>L</given-names></name><name><surname>Ma</surname><given-names>C</given-names></name><name><surname>Liu</surname><given-names>Q</given-names></name><name><surname>Weng</surname><given-names>H-J</given-names></name><name><surname>Cui</surname><given-names>Y</given-names></name><name><surname>Tang</surname><given-names>Z</given-names></name><name><surname>Kim</surname><given-names>Y</given-names></name><name><surname>Nie</surname><given-names>H</given-names></name><name><surname>Qu</surname><given-names>L</given-names></name><name><surname>Patel</surname><given-names>KN</given-names></name><name><surname>Li</surname><given-names>Z</given-names></name><name><surname>McNeil</surname><given-names>B</given-names></name><name><surname>He</surname><given-names>S</given-names></name><name><surname>Guan</surname><given-names>Y</given-names></name><name><surname>Xiao</surname><given-names>B</given-names></name><name><surname>Lamotte</surname><given-names>RH</given-names></name><name><surname>Dong</surname><given-names>X</given-names></name></person-group><year iso-8601-date="2013">2013</year><article-title>A subpopulation of nociceptors specifically linked to itch</article-title><source>Nature Neuroscience</source><volume>16</volume><fpage>174</fpage><lpage>182</lpage><pub-id pub-id-type="doi">10.1038/nn.3289</pub-id><pub-id pub-id-type="pmid">23263443</pub-id></element-citation></ref><ref id="bib15"><element-citation publication-type="confproc"><person-group person-group-type="author"><name><surname>He</surname><given-names>K</given-names></name><name><surname>Zhang</surname><given-names>X</given-names></name><name><surname>Ren</surname><given-names>S</given-names></name><name><surname>Sun</surname><given-names>J</given-names></name></person-group><year iso-8601-date="2016">2016</year><article-title>Deep residual learning for image recognition</article-title><conf-name>2016 IEEE Conference on Computer Vision and Pattern Recognition (CVPR</conf-name><pub-id pub-id-type="doi">10.1109/CVPR.2016.90</pub-id></element-citation></ref><ref id="bib16"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Hong</surname><given-names>J</given-names></name><name><surname>Buddenkotte</surname><given-names>J</given-names></name><name><surname>Berger</surname><given-names>TG</given-names></name><name><surname>Steinhoff</surname><given-names>M</given-names></name></person-group><year iso-8601-date="2011">2011</year><article-title>Management of itch in atopic dermatitis</article-title><source>Seminars in Cutaneous Medicine and Surgery</source><volume>30</volume><fpage>71</fpage><lpage>86</lpage><pub-id pub-id-type="doi">10.1016/j.sder.2011.05.002</pub-id><pub-id pub-id-type="pmid">21767767</pub-id></element-citation></ref><ref id="bib17"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Ikoma</surname><given-names>A</given-names></name><name><surname>Steinhoff</surname><given-names>M</given-names></name><name><surname>Ständer</surname><given-names>S</given-names></name><name><surname>Yosipovitch</surname><given-names>G</given-names></name><name><surname>Schmelz</surname><given-names>M</given-names></name></person-group><year iso-8601-date="2006">2006</year><article-title>The neurobiology of itch</article-title><source>Nature Reviews. Neuroscience</source><volume>7</volume><fpage>535</fpage><lpage>547</lpage><pub-id pub-id-type="doi">10.1038/nrn1950</pub-id><pub-id pub-id-type="pmid">16791143</pub-id></element-citation></ref><ref id="bib18"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Kobayashi</surname><given-names>K</given-names></name><name><surname>Matsushita</surname><given-names>S</given-names></name><name><surname>Shimizu</surname><given-names>N</given-names></name><name><surname>Masuko</surname><given-names>S</given-names></name><name><surname>Yamamoto</surname><given-names>M</given-names></name><name><surname>Murata</surname><given-names>T</given-names></name></person-group><year iso-8601-date="2021">2021</year><article-title>Automated detection of mouse scratching behaviour using convolutional recurrent neural network</article-title><source>Scientific Reports</source><volume>11</volume><elocation-id>658</elocation-id><pub-id pub-id-type="doi">10.1038/s41598-020-79965-w</pub-id><pub-id pub-id-type="pmid">33436724</pub-id></element-citation></ref><ref id="bib19"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Kremer</surname><given-names>AE</given-names></name><name><surname>Mettang</surname><given-names>T</given-names></name><name><surname>Weisshaar</surname><given-names>E</given-names></name></person-group><year iso-8601-date="2020">2020</year><article-title>Non-dermatological challenges of chronic itch</article-title><source>Acta Dermato-Venereologica</source><volume>100</volume><elocation-id>adv00025</elocation-id><pub-id pub-id-type="doi">10.2340/00015555-3345</pub-id><pub-id pub-id-type="pmid">31940045</pub-id></element-citation></ref><ref id="bib20"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Liu</surname><given-names>Q</given-names></name><name><surname>Tang</surname><given-names>Z</given-names></name><name><surname>Surdenikova</surname><given-names>L</given-names></name><name><surname>Kim</surname><given-names>S</given-names></name><name><surname>Patel</surname><given-names>KN</given-names></name><name><surname>Kim</surname><given-names>A</given-names></name><name><surname>Ru</surname><given-names>F</given-names></name><name><surname>Guan</surname><given-names>Y</given-names></name><name><surname>Weng</surname><given-names>H-J</given-names></name><name><surname>Geng</surname><given-names>Y</given-names></name><name><surname>Undem</surname><given-names>BJ</given-names></name><name><surname>Kollarik</surname><given-names>M</given-names></name><name><surname>Chen</surname><given-names>Z-F</given-names></name><name><surname>Anderson</surname><given-names>DJ</given-names></name><name><surname>Dong</surname><given-names>X</given-names></name></person-group><year iso-8601-date="2009">2009</year><article-title>Sensory neuron-specific GPCR mrgprs are itch receptors mediating chloroquine-induced pruritus</article-title><source>Cell</source><volume>139</volume><fpage>1353</fpage><lpage>1365</lpage><pub-id pub-id-type="doi">10.1016/j.cell.2009.11.034</pub-id><pub-id pub-id-type="pmid">20004959</pub-id></element-citation></ref><ref id="bib21"><element-citation publication-type="confproc"><person-group person-group-type="author"><name><surname>Liu</surname><given-names>Q</given-names></name><name><surname>Dong</surname><given-names>X</given-names></name></person-group><year iso-8601-date="2015">2015</year><article-title>The role of the Mrgpr receptor family in itch</article-title><conf-name>Pharmacology of itch</conf-name><fpage>71</fpage><lpage>88</lpage></element-citation></ref><ref id="bib22"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Loew</surname><given-names>ER</given-names></name><name><surname>MacMILLAN</surname><given-names>R</given-names></name><name><surname>Kaiser</surname><given-names>ME</given-names></name></person-group><year iso-8601-date="1946">1946</year><article-title>The anti-histamine properties of benadryl, beta-di-methylaminoethyl benzhydryl ether hydrochloride</article-title><source>The Journal of Pharmacology and Experimental Therapeutics</source><volume>86</volume><fpage>229</fpage><lpage>238</lpage><pub-id pub-id-type="pmid">21020043</pub-id></element-citation></ref><ref id="bib23"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Mathis</surname><given-names>A</given-names></name><name><surname>Mamidanna</surname><given-names>P</given-names></name><name><surname>Cury</surname><given-names>KM</given-names></name><name><surname>Abe</surname><given-names>T</given-names></name><name><surname>Murthy</surname><given-names>VN</given-names></name><name><surname>Mathis</surname><given-names>MW</given-names></name><name><surname>Bethge</surname><given-names>M</given-names></name></person-group><year iso-8601-date="2018">2018</year><article-title>DeepLabCut: markerless pose estimation of user-defined body parts with deep learning</article-title><source>Nature Neuroscience</source><volume>21</volume><fpage>1281</fpage><lpage>1289</lpage><pub-id pub-id-type="doi">10.1038/s41593-018-0209-y</pub-id><pub-id pub-id-type="pmid">30127430</pub-id></element-citation></ref><ref id="bib24"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Matterne</surname><given-names>U</given-names></name><name><surname>Strassner</surname><given-names>T</given-names></name><name><surname>Apfelbacher</surname><given-names>CJ</given-names></name><name><surname>Diepgen</surname><given-names>TL</given-names></name><name><surname>Weisshaar</surname><given-names>E</given-names></name></person-group><year iso-8601-date="2009">2009</year><article-title>Measuring the prevalence of chronic itch in the general population: development and validation of a questionnaire for use in large-scale studies</article-title><source>Acta Dermato-Venereologica</source><volume>89</volume><fpage>250</fpage><lpage>256</lpage><pub-id pub-id-type="doi">10.2340/00015555-0641</pub-id><pub-id pub-id-type="pmid">19479120</pub-id></element-citation></ref><ref id="bib25"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Morita</surname><given-names>T</given-names></name><name><surname>McClain</surname><given-names>SP</given-names></name><name><surname>Batia</surname><given-names>LM</given-names></name><name><surname>Pellegrino</surname><given-names>M</given-names></name><name><surname>Wilson</surname><given-names>SR</given-names></name><name><surname>Kienzler</surname><given-names>MA</given-names></name><name><surname>Lyman</surname><given-names>K</given-names></name><name><surname>Olsen</surname><given-names>ASB</given-names></name><name><surname>Wong</surname><given-names>JF</given-names></name><name><surname>Stucky</surname><given-names>CL</given-names></name><name><surname>Brem</surname><given-names>RB</given-names></name><name><surname>Bautista</surname><given-names>DM</given-names></name></person-group><year iso-8601-date="2015">2015</year><article-title>HTR7 mediates serotonergic acute and chronic itch</article-title><source>Neuron</source><volume>87</volume><fpage>124</fpage><lpage>138</lpage><pub-id pub-id-type="doi">10.1016/j.neuron.2015.05.044</pub-id><pub-id pub-id-type="pmid">26074006</pub-id></element-citation></ref><ref id="bib26"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Mu</surname><given-names>D</given-names></name><name><surname>Deng</surname><given-names>J</given-names></name><name><surname>Liu</surname><given-names>KF</given-names></name><name><surname>Wu</surname><given-names>ZY</given-names></name><name><surname>Shi</surname><given-names>YF</given-names></name><name><surname>Guo</surname><given-names>WM</given-names></name><name><surname>Mao</surname><given-names>QQ</given-names></name><name><surname>Liu</surname><given-names>XJ</given-names></name><name><surname>Li</surname><given-names>H</given-names></name><name><surname>Sun</surname><given-names>YG</given-names></name></person-group><year iso-8601-date="2017">2017</year><article-title>A central neural circuit for itch sensation</article-title><source>Science</source><volume>357</volume><fpage>695</fpage><lpage>699</lpage><pub-id pub-id-type="doi">10.1126/science.aaf4918</pub-id><pub-id pub-id-type="pmid">28818946</pub-id></element-citation></ref><ref id="bib27"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Park</surname><given-names>I</given-names></name><name><surname>Lee</surname><given-names>K</given-names></name><name><surname>Bishayee</surname><given-names>K</given-names></name><name><surname>Jeon</surname><given-names>HJ</given-names></name><name><surname>Lee</surname><given-names>H</given-names></name><name><surname>Lee</surname><given-names>U</given-names></name></person-group><year iso-8601-date="2019">2019</year><article-title>Machine-learning based automatic and real-time detection of mouse scratching behaviors</article-title><source>Experimental Neurobiology</source><volume>28</volume><fpage>54</fpage><lpage>61</lpage><pub-id pub-id-type="doi">10.5607/en.2019.28.1.54</pub-id><pub-id pub-id-type="pmid">30853824</pub-id></element-citation></ref><ref id="bib28"><element-citation publication-type="preprint"><person-group person-group-type="author"><name><surname>Paszke</surname><given-names>A</given-names></name><name><surname>Gross</surname><given-names>S</given-names></name><name><surname>Massa</surname><given-names>F</given-names></name><name><surname>Lerer</surname><given-names>A</given-names></name><name><surname>Bradbury</surname><given-names>J</given-names></name><name><surname>Chanan</surname><given-names>G</given-names></name><name><surname>Killeen</surname><given-names>T</given-names></name><name><surname>Lin</surname><given-names>Z</given-names></name><name><surname>Gimelshein</surname><given-names>N</given-names></name><name><surname>Antiga</surname><given-names>L</given-names></name></person-group><year iso-8601-date="2019">2019</year><article-title>Pytorch: An Imperative Style, High-Performance Deep Learning Library</article-title><source>arXiv</source><ext-link ext-link-type="uri" xlink:href="https://arxiv.org/abs/1912.01703">https://arxiv.org/abs/1912.01703</ext-link></element-citation></ref><ref id="bib29"><element-citation publication-type="preprint"><person-group person-group-type="author"><name><surname>Powers</surname><given-names>DM</given-names></name></person-group><year iso-8601-date="2020">2020</year><article-title>Evaluation: From Precision, Recall and F-Measure to ROC, Informedness, Markedness and Correlation</article-title><source>arXiv</source><ext-link ext-link-type="uri" xlink:href="https://arxiv.org/abs/2010.16061">https://arxiv.org/abs/2010.16061</ext-link></element-citation></ref><ref id="bib30"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Qu</surname><given-names>L</given-names></name><name><surname>Fu</surname><given-names>K</given-names></name><name><surname>Yang</surname><given-names>J</given-names></name><name><surname>Shimada</surname><given-names>SG</given-names></name><name><surname>LaMotte</surname><given-names>RH</given-names></name></person-group><year iso-8601-date="2015">2015</year><article-title>Cxcr3 chemokine receptor signaling mediates itch in experimental allergic contact dermatitis</article-title><source>Pain</source><volume>156</volume><fpage>1737</fpage><lpage>1746</lpage><pub-id pub-id-type="doi">10.1097/j.pain.0000000000000208</pub-id><pub-id pub-id-type="pmid">25932692</pub-id></element-citation></ref><ref id="bib31"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Sakamoto</surname><given-names>N</given-names></name><name><surname>Haraguchi</surname><given-names>T</given-names></name><name><surname>Kobayashi</surname><given-names>K</given-names></name><name><surname>Miyazaki</surname><given-names>Y</given-names></name><name><surname>Murata</surname><given-names>T</given-names></name></person-group><year iso-8601-date="2022">2022</year><article-title>Automated scratching detection system for black mouse using deep learning</article-title><source>Frontiers in Physiology</source><volume>13</volume><elocation-id>939281</elocation-id><pub-id pub-id-type="doi">10.3389/fphys.2022.939281</pub-id><pub-id pub-id-type="pmid">35936901</pub-id></element-citation></ref><ref id="bib32"><element-citation publication-type="confproc"><person-group person-group-type="author"><name><surname>Selvaraju</surname><given-names>RR</given-names></name><name><surname>Cogswell</surname><given-names>M</given-names></name><name><surname>Das</surname><given-names>A</given-names></name><name><surname>Vedantam</surname><given-names>R</given-names></name><name><surname>Parikh</surname><given-names>D</given-names></name><name><surname>Batra</surname><given-names>D</given-names></name></person-group><year iso-8601-date="2017">2017</year><article-title>Grad-CAM: Visual Explanations from Deep Networks via Gradient-Based Localization</article-title><conf-name>2017 IEEE International Conference on Computer Vision (ICCV</conf-name><conf-loc>Venice</conf-loc><pub-id pub-id-type="doi">10.1109/ICCV.2017.74</pub-id></element-citation></ref><ref id="bib33"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Shimada</surname><given-names>SG</given-names></name><name><surname>LaMotte</surname><given-names>RH</given-names></name></person-group><year iso-8601-date="2008">2008</year><article-title>Behavioral differentiation between itch and pain in mouse</article-title><source>Pain</source><volume>139</volume><fpage>681</fpage><lpage>687</lpage><pub-id pub-id-type="doi">10.1016/j.pain.2008.08.002</pub-id><pub-id pub-id-type="pmid">18789837</pub-id></element-citation></ref><ref id="bib34"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Solinski</surname><given-names>HJ</given-names></name><name><surname>Kriegbaum</surname><given-names>MC</given-names></name><name><surname>Tseng</surname><given-names>PY</given-names></name><name><surname>Earnest</surname><given-names>TW</given-names></name><name><surname>Gu</surname><given-names>X</given-names></name><name><surname>Barik</surname><given-names>A</given-names></name><name><surname>Chesler</surname><given-names>AT</given-names></name><name><surname>Hoon</surname><given-names>MA</given-names></name></person-group><year iso-8601-date="2019">2019</year><article-title>Nppb neurons are sensors of mast cell-induced itch</article-title><source>Cell Reports</source><volume>26</volume><fpage>3561</fpage><lpage>3573</lpage><pub-id pub-id-type="doi">10.1016/j.celrep.2019.02.089</pub-id><pub-id pub-id-type="pmid">30917312</pub-id></element-citation></ref><ref id="bib35"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Ständer</surname><given-names>S</given-names></name><name><surname>Weisshaar</surname><given-names>E</given-names></name><name><surname>Mettang</surname><given-names>T</given-names></name><name><surname>Szepietowski</surname><given-names>JC</given-names></name><name><surname>Carstens</surname><given-names>E</given-names></name><name><surname>Ikoma</surname><given-names>A</given-names></name><name><surname>Bergasa</surname><given-names>NV</given-names></name><name><surname>Gieler</surname><given-names>U</given-names></name><name><surname>Misery</surname><given-names>L</given-names></name><name><surname>Wallengren</surname><given-names>J</given-names></name><name><surname>Darsow</surname><given-names>U</given-names></name><name><surname>Streit</surname><given-names>M</given-names></name><name><surname>Metze</surname><given-names>D</given-names></name><name><surname>Luger</surname><given-names>TA</given-names></name><name><surname>Greaves</surname><given-names>MW</given-names></name><name><surname>Schmelz</surname><given-names>M</given-names></name><name><surname>Yosipovitch</surname><given-names>G</given-names></name><name><surname>Bernhard</surname><given-names>JD</given-names></name></person-group><year iso-8601-date="2007">2007</year><article-title>Clinical classification of itch: a position paper of the International forum for the study of itch</article-title><source>ACTA Dermato-Venereologica</source><volume>87</volume><fpage>291</fpage><lpage>294</lpage><pub-id pub-id-type="doi">10.2340/00015555-0305</pub-id><pub-id pub-id-type="pmid">17598029</pub-id></element-citation></ref><ref id="bib36"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Sun</surname><given-names>YG</given-names></name><name><surname>Chen</surname><given-names>ZF</given-names></name></person-group><year iso-8601-date="2007">2007</year><article-title>A gastrin-releasing peptide receptor mediates the itch sensation in the spinal cord</article-title><source>Nature</source><volume>448</volume><fpage>700</fpage><lpage>703</lpage><pub-id pub-id-type="doi">10.1038/nature06029</pub-id><pub-id pub-id-type="pmid">17653196</pub-id></element-citation></ref><ref id="bib37"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Thurmond</surname><given-names>RL</given-names></name><name><surname>Gelfand</surname><given-names>EW</given-names></name><name><surname>Dunford</surname><given-names>PJ</given-names></name></person-group><year iso-8601-date="2008">2008</year><article-title>The role of histamine H1 and H4 receptors in allergic inflammation: the search for new antihistamines</article-title><source>Nature Reviews. Drug Discovery</source><volume>7</volume><fpage>41</fpage><lpage>53</lpage><pub-id pub-id-type="doi">10.1038/nrd2465</pub-id><pub-id pub-id-type="pmid">18172439</pub-id></element-citation></ref><ref id="bib38"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Weisshaar</surname><given-names>E</given-names></name><name><surname>Dalgard</surname><given-names>F</given-names></name></person-group><year iso-8601-date="2009">2009</year><article-title>Epidemiology of itch: adding to the burden of skin morbidity</article-title><source>Acta Dermato-Venereologica</source><volume>89</volume><fpage>339</fpage><lpage>350</lpage><pub-id pub-id-type="doi">10.2340/00015555-0662</pub-id><pub-id pub-id-type="pmid">19688144</pub-id></element-citation></ref><ref id="bib39"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Wimalasena</surname><given-names>NK</given-names></name><name><surname>Milner</surname><given-names>G</given-names></name><name><surname>Silva</surname><given-names>R</given-names></name><name><surname>Vuong</surname><given-names>C</given-names></name><name><surname>Zhang</surname><given-names>Z</given-names></name><name><surname>Bautista</surname><given-names>DM</given-names></name><name><surname>Woolf</surname><given-names>CJ</given-names></name></person-group><year iso-8601-date="2021">2021</year><article-title>Dissecting the precise nature of itch-evoked scratching</article-title><source>Neuron</source><volume>109</volume><fpage>3075</fpage><lpage>3087</lpage><pub-id pub-id-type="doi">10.1016/j.neuron.2021.07.020</pub-id><pub-id pub-id-type="pmid">34411514</pub-id></element-citation></ref><ref id="bib40"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Yosipovitch</surname><given-names>G</given-names></name><name><surname>Fleischer</surname><given-names>AB</given-names></name></person-group><year iso-8601-date="2003">2003</year><article-title>Itch associated with skin disease</article-title><source>American Journal of Clinical Dermatology</source><volume>4</volume><fpage>617</fpage><lpage>622</lpage><pub-id pub-id-type="doi">10.2165/00128071-200304090-00004</pub-id><pub-id pub-id-type="pmid">12926980</pub-id></element-citation></ref><ref id="bib41"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Yosipovitch</surname><given-names>G</given-names></name><name><surname>Rosen</surname><given-names>JD</given-names></name><name><surname>Hashimoto</surname><given-names>T</given-names></name></person-group><year iso-8601-date="2018">2018</year><article-title>Itch: from mechanism to (novel) therapeutic approaches</article-title><source>The Journal of Allergy and Clinical Immunology</source><volume>142</volume><fpage>1375</fpage><lpage>1390</lpage><pub-id pub-id-type="doi">10.1016/j.jaci.2018.09.005</pub-id><pub-id pub-id-type="pmid">30409247</pub-id></element-citation></ref><ref id="bib42"><element-citation publication-type="journal"><person-group person-group-type="author"><name><surname>Yu</surname><given-names>H</given-names></name><name><surname>Wangensteen</surname><given-names>K</given-names></name><name><surname>Deng</surname><given-names>T</given-names></name><name><surname>Li</surname><given-names>Y</given-names></name><name><surname>Luo</surname><given-names>W</given-names></name></person-group><year iso-8601-date="2021">2021</year><article-title>MRGPRX4 in cholestatic pruritus</article-title><source>Seminars in Liver Disease</source><volume>41</volume><fpage>358</fpage><lpage>367</lpage><pub-id pub-id-type="doi">10.1055/s-0041-1730923</pub-id><pub-id pub-id-type="pmid">34161994</pub-id></element-citation></ref></ref-list></back><sub-article article-type="editor-report" id="sa0"><front-stub><article-id pub-id-type="doi">10.7554/eLife.84042.sa0</article-id><title-group><article-title>Editor's evaluation</article-title></title-group><contrib-group><contrib contrib-type="author"><name><surname>Kim</surname><given-names>Brian S</given-names></name><role specific-use="editor">Reviewing Editor</role><aff><institution-wrap><institution-id institution-id-type="ror">https://ror.org/04a9tmd77</institution-id><institution>Icahn School of Medicine at Mount Sinai</institution></institution-wrap><country>United States</country></aff></contrib></contrib-group></front-stub><body><p>Scratch assays are the gold standard for measuring itch in rodents. However, the current limitation is that this is performed manually which is enormously taxing in terms of hours spent counting scratching bouts. The authors have developed a valuable automatic system to quantify scratch behavior with high accuracy and provided a valuable tool for the field. This will be resourceful for the greater itch biology community.</p></body></sub-article><sub-article article-type="decision-letter" id="sa1"><front-stub><article-id pub-id-type="doi">10.7554/eLife.84042.sa1</article-id><title-group><article-title>Decision letter</article-title></title-group><contrib-group content-type="section"><contrib contrib-type="editor"><name><surname>Kim</surname><given-names>Brian S</given-names></name><role>Reviewing Editor</role><aff><institution-wrap><institution-id institution-id-type="ror">https://ror.org/04a9tmd77</institution-id><institution>Icahn School of Medicine at Mount Sinai</institution></institution-wrap><country>United States</country></aff></contrib></contrib-group></front-stub><body><boxed-text id="sa2-box1"><p>In the interests of transparency, eLife publishes the most substantive revision requests and the accompanying author responses.</p></boxed-text><p><bold>Decision letter after peer review:</bold></p><p>Thank you for submitting your article &quot; Scratch-AID: A Deep-learning Based System for Automatic Detection of Mouse Scratching Behavior with High Accuracy&quot; for consideration by eLife. Your article has been reviewed by 2 peer reviewers, and the evaluation has been overseen by a Reviewing Editor and Catherine Dulca as the Senior Editor. The reviewers have opted to remain anonymous.</p><p><italic>Reviewer #1 (Recommendations for the authors):</italic></p><p>1. Can Scratch-AID be modified to quantify scratching bouts?</p><p>2. Saliency maps analysis suggest that the trained neural network is focusing on the movement of the hindleg to identify scratching. Although mice will spend the majority of their time scratching the injected or treated skin location, they do scratch uninjected/untreated area as well in both acute and chronic itch models. Different laboratories have different strategies to determine whether scratching to the untreated locations is included for quantification. Is Scratch-AID able to differentiate scratching directed towards treated vs untreated locations?</p><p><italic>Reviewer #2 (Recommendations for the authors):</italic></p><p>Mouse behavior assay is the primary readout for almost all itch research. The itch sensation triggers a stereotypical scratch-lick response by the ipsilateral hind paw, and several key parameters including the number of scratch bouts and total scratching duration can be precisely quantified and compared across genotypes and treatment groups. So far, the vast majority of itch behavior videos are manually scored, requiring hundreds of hours of labor and are prone to experimenter errors. In this paper, the authors introduce an automated system to replace the manual itch counting process. First, they designed a standardized videotaping box to produce consistent and high-quality videos of itch behaviors. They then annotated the videos frame by frame for scratch bouts, and trained a neural network to identify scratching behaviors with high precision. The system (named Scratch-AID by the authors) performed consistently with manual and reference scoring, and yielded key quantitative parameters (scratching bout counts and total scratching times). Saliency map revealed that the trained neural network focused on the scratching hind paw, which is consistent with what human observers look at. The authors further showed that Scratch-AID can be easily adapted to quantify scratching at slightly different sites (nape vs cheek), in response to different itch mediators (chloroquine vs histamine), in both acute and chronic itch models, and that it successfully revealed the effect of a generic anti-itch drug Benadryl. Overall, the model appears accurate and easily adaptable and can probably be widely used by the itch field.</p><p>There had been a few previous attempts to train neural networks to score itch behaviors (Kobayashi et al., 2021; Sakamoto et al., 2022). The main advance of this paper is the incorporation of a standardized recording system, and larger number of videos in the training dataset. The authors provided very detailed and thorough analyses of their own model, but did not compare its sensitivity and precision to the previous models. It is also unclear why the authors did not choose the high-speed video system that they previously published for detailed analyses of pain behaviors (Abdus-Saboor et al., 2019), despite evidence that multiple itch parameters can only be captured at high speed (Wimalasena, et al. 2021). The authors show that scratch-AID can accurately score multiple itch models (despite it being trained on chloroquine nape injection videos), but does not show whether it can distinguish itch behaviors induced by different mechanisms and mediators. For example, whether it can detect activation of different itch neuron populations (termed NP1, NP2 and NP3 by single cell transcriptomic analysis), or distinguish direct itch neuron activation by mediators such as chloroquine vs mast cell activators such as 48/80. Nonetheless, this study provides a great opportunity for the itch field to standardize behavior data acquisition and analysis, and will greatly enhance the throughput and quality of future itch behavior data.</p><p>The data are overall very convincing and the system an exciting advancement that will probably be widely used. I have some questions regarding data analysis and potential concerns for future users of the technology.</p><p>1. Does the model distinguish ipsilateral vs contralateral scratches?</p><p>2. Can the authors present the data in more detail? For example, present and compare both scratch bout count and total scratching time between scratch-AID and manual scoring for each of the itch models.</p><p>3. This is probably a given but can the authors provide evidence that the model clearly distinguishes pain (by capsaicin injection) vs itch (chloroquine and histamine)?</p><p>4. If all parameters are analyzed in more detail (average duration of scratch bouts, temporal distribution of scratches across the imaging period), can the model distinguish the differences between NP1(by β-alanine), NP2(by chloroquine) and NP3(by 5-HT) activation?</p><p>5. Can the model distinguish direct neuronal activation (chloroquine) vs immune cell activation (48/80, anti-IgE)?</p><p>6. Since the training is done with black mice, can the model detect itch behavior by white mice?</p><p>7. Can the authors comment on how the new system compares to previous itch-counting apparatus (such as SCLABA-REAL) and previous neural network models?</p><p>8. What's the max number of animals that can be recorded and analyzed in a single box?</p><p>9. Since video quality is vital for correct predictions by neural networks, can the authors comment on the influence of potential confounding factors (urine and feces, wear and tear of the acrylic imaging surface).</p><p>10. This is potentially a great opportunity for the itch field to standardize the itch behavior assay. Can the authors comment briefly in the discussion on other aspects of standardization such as time of the day for experiments (circadian rhythm), injection volumes, habituation, and sexual dimorphism?</p><p>11. Can the authors discuss how the rest of the field can apply the system in their own research? Will it be released as an open source software? And will the custom video box be commercially available?</p></body></sub-article><sub-article article-type="reply" id="sa2"><front-stub><article-id pub-id-type="doi">10.7554/eLife.84042.sa2</article-id><title-group><article-title>Author response</article-title></title-group></front-stub><body><disp-quote content-type="editor-comment"><p>Reviewer #1 (Recommendations for the authors):</p><p>1. Can Scratch-AID be modified to quantify scratching bouts?</p></disp-quote><p>We thank the reviewer for asking the important question. Our current model cannot directly quantify scratching bouts. Nevertheless, since the high correlation between the total number of scratching bouts and total scratching time, the total scratching bouts can be estimated based on the total scratching time and average duration of each scratching bout in a specific itch model. In addition, with more detailed manual annotation of scratching bouts in each scratching train, it is possible to train a neural network to recognize and quantify scratching bouts in the future.</p><disp-quote content-type="editor-comment"><p>2. Saliency maps analysis suggest that the trained neural network is focusing on the movement of the hindleg to identify scratching. Although mice will spend the majority of their time scratching the injected or treated skin location, they do scratch uninjected/untreated area as well in both acute and chronic itch models. Different laboratories have different strategies to determine whether scratching to the untreated locations is included for quantification. Is Scratch-AID able to differentiate scratching directed towards treated vs untreated locations?</p></disp-quote><p>We appreciate the reviewer’s interesting comments and questions. Our current model doesn’t distinguish the scratching location. We viewed it as a strength that Scratch-AID can be generalized to recognize scratching in different body locations and itch models. This could become a problem if the number of off-target scratching makes a significant portion of the total scratching and obscures the real phenotype. In this case, the researchers may want to double check the videos and distinguish the scratching location, such as ipsilateral vs. contralateral, or cheek vs. nape. In principle, with more training videos containing different scratching locations and more detailed manual annotations, a neural network can be trained to only recognize scratching towards a particular body location.</p><disp-quote content-type="editor-comment"><p>Reviewer #2 (Recommendations for the authors):</p><p>[…]</p><p>The data are overall very convincing and the system an exciting advancement that will probably be widely used. I have some questions regarding data analysis and potential concerns for future users of the technology.</p><p>1. Does the model distinguish ipsilateral vs contralateral scratches?</p></disp-quote><p>Our current models cannot distinguish ipsilateral vs contralateral scratches. In principle, with the manual annotation of left or right-side scratching, a neural network can be trained to distinguish left vs right scratching. This could be an interesting future direction.</p><disp-quote content-type="editor-comment"><p>2. Can the authors present the data in more detail? For example, present and compare both scratch bout count and total scratching time between scratch-AID and manual scoring for each of the itch models.</p></disp-quote><p>We systematically compared the total scratching time between scratch-AID and manual scoring for different itch models (Fig. 3, Fig. 5 and Fig. 6). Our current model cannot count the scratch bouts. As discussed above (answers to question #1 of reviewer #1), it will take additional manual annotation to train the neural network for this purpose.</p><disp-quote content-type="editor-comment"><p>3. This is probably a given but can the authors provide evidence that the model clearly distinguishes pain (by capsaicin injection) vs itch (chloroquine and histamine)?</p></disp-quote><p>Our model can distinguish itch-induced scratching behavior and pain-related wiping behavior. In all test videos, no wiping behavior was recognized as scratching behavior. We showed examples that the wiping behavior and other behaviors were not recognized as scratching behavior (false positive, type I error, Figure 3—figure supplement 4).</p><disp-quote content-type="editor-comment"><p>4. If all parameters are analyzed in more detail (average duration of scratch bouts, temporal distribution of scratches across the imaging period), can the model distinguish the differences between NP1(by β-alanine), NP2(by chloroquine) and NP3(by 5-HT) activation?</p></disp-quote><p>Our current model recognizes scratching trains. Thus, it can display the temporal distribution of scratching trains across the imaging period, quantify scratching train durations or average duration, but cannot analyze the average duration of scratching bouts (as the model can’t quantify the scratching bouts). If the temporal distribution of scratching trains or scratching train durations are different between NP1, NP2 and NP3 neuron mediated itch, our model may distinguish the differences.</p><disp-quote content-type="editor-comment"><p>5. Can the model distinguish direct neuronal activation (chloroquine) vs immune cell activation (48/80, anti-IgE)?</p></disp-quote><p>As discussed above, our current model can quantify the total scratching time, display the temporal distribution of scratching trains, and quantify scratching train durations. If these features show differences in scratching behavior induced by neuronal activation vs immune cell activation, for example the direct neuronal activation might trigger scratching behaviors in a shorter delay than the immune cell activation, then our model could be able to distinguish direct neuronal activation vs immune cell activation.</p><disp-quote content-type="editor-comment"><p>6. Since the training is done with black mice, can the model detect itch behavior by white mice?</p></disp-quote><p>We haven’t tested our model using white mice. Since the videotaping box is white color, the video quality for white mice might decrease due to the low contrast. Testing Scratch-AID with mice of different coat colors could be a future direction. If the contrast is the problem, one option is to change to box color for better contrast.</p><disp-quote content-type="editor-comment"><p>7. Can the authors comment on how the new system compares to previous itch-counting apparatus (such as SCLABA-REAL) and previous neural network models?</p></disp-quote><p>We appreciate the reviewer raising the question. However, it’s hard for us to do the side-by-side comparison between our model and SCLABA-REAL, because there are no published data about performance of SCLABA-REAL. According to the product information, SCLABA-REAL records the video from top view, and uses frame-to-frame subtraction to recognize the scratching behavior in a real time. Our Scratch-AID records the video from bottom view and uses raw frames to recognize the scratching behavior in an off-line manner.</p><p>Compared to previous neural network models, the main novelties of our study include: (1) designed of a videotaping box for stable and reproducible video recording in different labs; (2) collected a large amount of frame-labeled dataset for the training; (3) optimized CRNN architecture and hyperparameters for more effective learning; and (4) tested the generalization of the model in major mouse itch models and in a drug screening paradigm.</p><disp-quote content-type="editor-comment"><p>8. What's the max number of animals that can be recorded and analyzed in a single box?</p></disp-quote><p>Only one mouse can be recorded and analyzed in a single box. However, with multiple boxes, several mice can be recorded simultaneously.</p><disp-quote content-type="editor-comment"><p>9. Since video quality is vital for correct predictions by neural networks, can the authors comment on the influence of potential confounding factors (urine and feces, wear and tear of the acrylic imaging surface).</p></disp-quote><p>This is a very good question. According to our experience, sufficient habituation greatly reduced the urine and feces. We also cleaned the floor (urine and feces) between two recordings. A small amount of urine and feces did not have significant influence on the performance of the model. We don’t think that wear and tear of the acrylic imaging surface will be a big issue. After around 200 video recordings, no obvious wear and tear of the acrylic imaging surface were observed. Even if there are some wear and tear after long-term use, it’s easy to replace the imaging floor with a new one (same material and quality).</p><disp-quote content-type="editor-comment"><p>10. This is potentially a great opportunity for the itch field to standardize the itch behavior assay. Can the authors comment briefly in the discussion on other aspects of standardization such as time of the day for experiments (circadian rhythm), injection volumes, habituation, and sexual dimorphism?</p></disp-quote><p>The reviewer raises an important point. As the reviewer mentioned, circadian rhythm, injection volumes, habituation, sexual dimorphism and other factors might affect the itch behavior assay and results. For our experiments, we did the scratching behavior recording during the daytime, injected 15 ul pruritogens dissolved in saline for both cheek and nape model, habituated each mouse in the recoding box 15 min/day for three consecutive days, and included both male and female mice. To standardize the mouse itch behavior assay, the itch field need to discuss together to reach a consensus.</p><disp-quote content-type="editor-comment"><p>11. Can the authors discuss how the rest of the field can apply the system in their own research? Will it be released as an open source software? And will the custom video box be commercially available?</p></disp-quote><p>We really hope that the rest of the mouse itch field will try and take advantage of our Scratch-AID system for their studies. This is the motivation for us to work on this project. Our videos, codes, and models will be publicly available as this manuscript is published. We would also like to further optimize our prototype models, add features as suggested by the reviewers, and launch a user-friendly software or a web-based analysis system (if we could have funding support and identified a software developer).</p></body></sub-article></article>