<!DOCTYPE article
PUBLIC "-//NLM//DTD JATS (Z39.96) Journal Archiving and Interchange DTD with MathML3 v1.3 20210610//EN" "JATS-archivearticle1-3-mathml3.dtd">
<article xmlns:mml="http://www.w3.org/1998/Math/MathML" xmlns:xlink="http://www.w3.org/1999/xlink" dtd-version="1.3" xml:lang="en" article-type="research-article"><?properties manuscript?><processing-meta base-tagset="archiving" mathml-version="3.0" table-model="xhtml" tagset-family="jats"><restricted-by>pmc</restricted-by></processing-meta><front><journal-meta><journal-id journal-id-type="nlm-journal-id">8703089</journal-id><journal-id journal-id-type="pubmed-jr-id">3587</journal-id><journal-id journal-id-type="nlm-ta">Epilepsy Res</journal-id><journal-id journal-id-type="iso-abbrev">Epilepsy Res</journal-id><journal-title-group><journal-title>Epilepsy research</journal-title></journal-title-group><issn pub-type="ppub">0920-1211</issn><issn pub-type="epub">1872-6844</issn></journal-meta><article-meta><article-id pub-id-type="pmid">35994859</article-id><article-id pub-id-type="pmc">9810436</article-id><article-id pub-id-type="doi">10.1016/j.eplepsyres.2022.107013</article-id><article-id pub-id-type="manuscript">NIHMS1861737</article-id><article-categories><subj-group subj-group-type="heading"><subject>Article</subject></subj-group></article-categories><title-group><article-title>A replicable, open-source, data integration method to support national practice-based research &#x00026; quality improvement systems</article-title></title-group><contrib-group><contrib contrib-type="author"><name><surname>Fernandes</surname><given-names>Marta</given-names></name><xref rid="A1" ref-type="aff">a</xref><xref rid="A2" ref-type="aff">b</xref><xref rid="A3" ref-type="aff">c</xref><xref rid="CR1" ref-type="corresp">*</xref></contrib><contrib contrib-type="author"><name><surname>Donahue</surname><given-names>Maria A.</given-names></name><xref rid="A1" ref-type="aff">a</xref><xref rid="A2" ref-type="aff">b</xref><xref rid="A4" ref-type="aff">d</xref></contrib><contrib contrib-type="author"><name><surname>Hoch</surname><given-names>Dan</given-names></name><xref rid="A1" ref-type="aff">a</xref><xref rid="A2" ref-type="aff">b</xref></contrib><contrib contrib-type="author"><name><surname>Cash</surname><given-names>Sydney</given-names></name><xref rid="A1" ref-type="aff">a</xref><xref rid="A2" ref-type="aff">b</xref></contrib><contrib contrib-type="author"><name><surname>Zafar</surname><given-names>Sahar</given-names></name><xref rid="A1" ref-type="aff">a</xref><xref rid="A2" ref-type="aff">b</xref></contrib><contrib contrib-type="author"><name><surname>Jacobs</surname><given-names>Claire</given-names></name><xref rid="A1" ref-type="aff">a</xref><xref rid="A2" ref-type="aff">b</xref></contrib><contrib contrib-type="author"><name><surname>Hosford</surname><given-names>Mackenzie</given-names></name><xref rid="A1" ref-type="aff">a</xref><xref rid="A2" ref-type="aff">b</xref></contrib><contrib contrib-type="author"><name><surname>Voinescu</surname><given-names>P. Emanuela</given-names></name><xref rid="A2" ref-type="aff">b</xref><xref rid="A5" ref-type="aff">e</xref></contrib><contrib contrib-type="author"><name><surname>Fureman</surname><given-names>Brandy</given-names></name><xref rid="A6" ref-type="aff">f</xref></contrib><contrib contrib-type="author"><name><surname>Buchhalter</surname><given-names>Jeffrey</given-names></name><xref rid="A7" ref-type="aff">g</xref></contrib><contrib contrib-type="author"><name><surname>McGraw</surname><given-names>Christopher Michael</given-names></name><xref rid="A1" ref-type="aff">a</xref><xref rid="A2" ref-type="aff">b</xref><xref rid="FN1" ref-type="author-notes">1</xref></contrib><contrib contrib-type="author"><name><surname>Westover</surname><given-names>M. Brandon</given-names></name><xref rid="A1" ref-type="aff">a</xref><xref rid="A2" ref-type="aff">b</xref><xref rid="A3" ref-type="aff">c</xref><xref rid="A8" ref-type="aff">h</xref><xref rid="FN1" ref-type="author-notes">1</xref></contrib><contrib contrib-type="author"><name><surname>Moura</surname><given-names>Lidia M.V.R.</given-names></name><xref rid="A1" ref-type="aff">a</xref><xref rid="A2" ref-type="aff">b</xref><xref rid="A4" ref-type="aff">d</xref><xref rid="FN1" ref-type="author-notes">1</xref></contrib></contrib-group><aff id="A1"><label>a</label>Department of Neurology, Massachusetts General Hospital (MGH), Boston, MA, United States</aff><aff id="A2"><label>b</label>Harvard Medical School, Boston, MA, United States</aff><aff id="A3"><label>c</label>Clinical Data Animation Center (CDAC), MGH, Boston, MA, United States</aff><aff id="A4"><label>d</label>The NeuroValue Lab, MGH, Boston, MA, United States</aff><aff id="A5"><label>e</label>Department of Neurology, Division of Epilepsy, Division of Women&#x02019;s Health, Brigham and Women&#x02019;s Hospital, Boston, MA, United States</aff><aff id="A6"><label>f</label>Epilepsy Foundation of America, United States</aff><aff id="A7"><label>g</label>Department of Pediatrics, University of Calgary School of Medicine, Calgary, Canada</aff><aff id="A8"><label>h</label>McCance Center for Brain Health, MGH, Boston, MA, United States</aff><author-notes><fn fn-type="con" id="FN1"><label>1</label><p id="P1">Co-senior authors</p></fn><corresp id="CR1"><label>*</label>Corresponding author at: Department of Neurology, Massachusetts General Hospital (MGH), Boston, MA, United States. <email>mbentofernandes@mgh.harvard.edu</email> (M. Fernandes)</corresp></author-notes><pub-date pub-type="nihms-submitted"><day>30</day><month>12</month><year>2022</year></pub-date><pub-date pub-type="ppub"><month>10</month><year>2022</year></pub-date><pub-date pub-type="epub"><day>18</day><month>8</month><year>2022</year></pub-date><pub-date pub-type="pmc-release"><day>03</day><month>1</month><year>2023</year></pub-date><volume>186</volume><fpage>107013</fpage><lpage>107013</lpage><abstract id="ABS1"><sec id="S1"><title>Objectives:</title><p id="P2">The Epilepsy Learning Healthcare System (ELHS) was created in 2018 to address measurable improvements in outcomes for people with epilepsy. However, fragmentation of data systems has been a major barrier for reporting and participation. In this study, we aimed to test the feasibility of an open-source Data Integration (DI) method that connects real-life clinical data to national research and quality improvement (QI) systems.</p></sec><sec id="S2"><title>Methods:</title><p id="P3">The ELHS case report forms were programmed as EPIC SmartPhrases at Mass General Brigham (MGB) in December 2018 and subsequently as EPIC SmartForms in June 2021 to collect actionable, standardized, structured epilepsy data in the electronic health record (EHR) for subsequent pull into the external national registry of the ELHS. Following the QI methodology in the Chronic Care Model, 39 providers, epileptologists and neurologists, incorporated the ELHS SmartPhrase into their clinical workflow, focusing on collecting diagnosis of epilepsy, seizure type according to the International League Against Epilepsy, seizure frequency, date of last seizure, medication adherence and side effects. The collected data was stored in the Enterprise Data Warehouse (EDW) without integration with external systems. We developed and validated a DI method that extracted the data from EDW using structured query language and later preprocessed using text mining. We used the ELHS data dictionary to match fields in the preprocessed notes to obtain the final structured dataset with seizure control information. For illustration, we described the data curated from the care period of 12/2018&#x02013;12/2021.</p></sec><sec id="S3"><title>Results:</title><p id="P4">The cohort comprised a total of 1806 patients with a mean age of 43 years old (SD: 17.0), where 57% were female, 80% were white, and 84% were non-Hispanic/Latino. Using our DI method, we automated the data mining, preprocessing, and exporting of the structured dataset into a local database, to be weekly accessible to clinicians and quality improvers. During the period of SmartPhrase implementation, there were 5168 clinic visits logged by providers documenting each patient&#x02019;s seizure type and frequency. During this period, providers documented 59% patients having focal seizures, 35% having generalized seizures and 6% patients having another type. Of the cohort, 45% patients had private insurance. The resulting structured dataset was bulk uploaded via web interface into the external national registry of the ELHS.</p></sec><sec id="S4"><title>Conclusions:</title><p id="P5">Structured data can be feasibly extracted from text notes of epilepsy patients for weekly reporting to a national learning healthcare system.</p></sec></abstract><kwd-group><kwd>Epilepsy</kwd><kwd>International League Against Epilepsy</kwd><kwd>Seizure control</kwd><kwd>Quality Improvement</kwd><kwd>Learning Health System</kwd><kwd>Text analytics</kwd></kwd-group></article-meta></front><body><sec id="S5"><label>1.</label><title>Introduction</title><p id="P6">Digital technology plays an increasingly critical role in healthcare organizations, enabling the reduction in the reporting burden for health care staff and facilitating the effective use of data for practice improvement, clinical decision making, and population management (<xref rid="R1" ref-type="bibr">Bell et al., 2018</xref>; <xref rid="R19" ref-type="bibr">Mesk&#x000f3; et al., 2017</xref>; <xref rid="R29" ref-type="bibr">Thimbleby, 2013</xref>). Especially in health information systems with the transition to electronic health records (EHR), the development of digital tools to help collect and report health data has proved to be a key need (<xref rid="R31" ref-type="bibr">World Health Organization, 2019b</xref>). However, there are many barriers to data driven healthcare improvement and several approaches have emerged including Learning Healthcare Systems (LHS) (<xref rid="R10" ref-type="bibr">Enticott et al., 2021</xref>). LHS use continuous Quality Improvement (QI) strategies to improve care over time (<xref rid="R20" ref-type="bibr">Morain et al., 2017</xref>). LHS models embed healthcare data-driven research, integrating multidisciplinary expertise to deliver improved health (<xref rid="R4" ref-type="bibr">Budrionis and Bellika, 2016</xref>; <xref rid="R17" ref-type="bibr">McLachlan et al., 2018</xref>; <xref rid="R18" ref-type="bibr">Menear et al., 2019</xref>; <xref rid="R26" ref-type="bibr">Scobie and Castle-Clarke, 2019</xref>), through data collection and analysis at the point of care to inform clinical decision making (<xref rid="R26" ref-type="bibr">Scobie and Castle-Clarke, 2019</xref>; <xref rid="R28" ref-type="bibr">Teede et al., 2019</xref>), thereby improving healthcare and outcomes (<xref rid="R10" ref-type="bibr">Enticott et al., 2021</xref>). Clinicians use common data elements (CDEs), which consist of standardized questions and answer choices that allow aggregation, analysis, and comparison of observations from multiple sources (<xref rid="R15" ref-type="bibr">Loring et al., 2011</xref>), and are foundational for LHS.</p><p id="P7">Even though LHS environments are producing successful impact across multiple continents and settings (<xref rid="R10" ref-type="bibr">Enticott et al., 2021</xref>), very few LHSs focus on neurological disorders (<xref rid="R2" ref-type="bibr">Bindman et al., 2018</xref>). The Epilepsy Learning Healthcare System (ELHS) was created in 2018 to improve epilepsy care at a system level by addressing measurable improvements in outcomes for people with epilepsy (<xref rid="R8" ref-type="bibr">Donahue et al., 2021</xref>). Epilepsy is a chronic disease accounting for 0.5% of the global burden of disease, a time-based measure that combines years of life lost due to premature mortality and time lived in less than full health (<xref rid="R30" ref-type="bibr">World Health Organization, 2019a</xref>). Epilepsy affects 50&#x02013;70 million people of all ages worldwide (<xref rid="R22" ref-type="bibr">Ngugi et al., 2010</xref>), being the fourth most common neurological disorder globally (<xref rid="R11" ref-type="bibr">Epilepsy Foundation, 2020b</xref>; <xref rid="R25" ref-type="bibr">Scheffer et al., 2017</xref>). Epilepsy has significant economic implications in terms of health-care needs, premature death and lost work productivity (<xref rid="R22" ref-type="bibr">Ngugi et al., 2010</xref>; <xref rid="R30" ref-type="bibr">World Health Organization, 2019a</xref>). Despite significant pharmacological and technological advances in the field of epilepsy, at least 3 out of 10 people with epilepsy continue to have seizures because available treatments do not completely control their seizures (<xref rid="R12" ref-type="bibr">Epilepsy Foundation, 2020a</xref>; <xref rid="R14" ref-type="bibr">Forsgren et al., 2005</xref>).</p><p id="P8">ELHS goals include the improvement of care processes and clinical outcomes by utilizing QI methodology in the context of a Chronic Care Model (<xref rid="R16" ref-type="bibr">Margolis et al., 2013</xref>). Once ELHS standardized epilepsy care metrics are being collected reliably for a vast number of patients, in addition to directly improving outcomes via QI methodology, the large numbers will facilitate comparative effectiveness studies and identification of patients for large real-world clinical trials. Seizure frequency documentation and screening for barriers to medication adherence were selected by ELHS as the first two network-wide interventions (<xref rid="R8" ref-type="bibr">Donahue et al., 2021</xref>).</p><p id="P9">The ELHS case report forms (CRFs) were designed to collect information in a standardized manner that is essential to patient care, useful for QI and feasible within the clinic workflow. The ELHS CRFs were programmed as a SmartPhrase in the EPIC EHR software at Mass General Brigham (MGB) in December 2018 to collect actionable, standardized, structured epilepsy data in the EHR for data integration (DI) and subsequent pull into the national registry of the ELHS. Epileptologists and neurologists incorporated the ELHS SmartPhrase into their clinical workflow, focusing in collecting seizure control information, which was stored as semi-structured text data in the MGB system database. Although the SmartPhrase allows physicians to enter data in a structured manner, the backend database stores the data as free text. Thus, to make use of this data for research and quality improvement processes, it is necessary to create an automated extraction method (<xref rid="R9" ref-type="bibr">Elbattah et al., 2021</xref>).</p><p id="P10">Text Analytics, or Text Mining, is generally defined as &#x0201c;the methodology followed to derive quality and actionable insights from textual data&#x0201d; (<xref rid="R24" ref-type="bibr">Sarkar, 2019</xref>). Text Mining represents a field of techniques and technologies that include machine learning, natural language processing (NLP), and information retrieval. Leveraging the power of text mining permits transformation of semi-structured and unstructured text data into structured information from which powerful insights can be derived. Thus, in this paper we present a text mining methodology to extract seizure control information from the ELHS EHR SmartPhrase and analysis of this data. Using our DI method, we automated the data mining, preprocessing, and exporting of the structured dataset into a local database, to be accessible on a weekly basis to clinicians and quality improvers. We uploaded this information to the national registry of ELHS, thereby promoting its development and growth.</p></sec><sec id="S6"><label>2.</label><title>Methods</title><sec id="S7"><label>2.1.</label><title>Study overview</title><p id="P11">The ELHS developed CRFs to collect standardized epilepsy care related data (<xref rid="R8" ref-type="bibr">Donahue et al., 2021</xref>). The ELHS CRFs were programmed as EPIC SmartPhrases at MGB in December 2018 and subsequently as EPIC SmartForms in June 2021 to collect actionable, standardized, structured epilepsy data in the EHR for subsequent pull into the external national registry of the ELHS. Following the QI methodology in the Chronic Care Model (<xref rid="R16" ref-type="bibr">Margolis et al., 2013</xref>), a total of 39 epileptologists and neurologists incorporated the ELHS SmartPhrase into their clinical workflow, focusing on collecting seizure type according to the International League Against Epilepsy (ILAE) seizure classification (<xref rid="R25" ref-type="bibr">Scheffer et al., 2017</xref>), seizure frequency, date of last seizure and anti-seizure medication (ASM) adherence and side effects. ASM information fields were only added in a later version of the SmartPhrase, introduced in July 2020.</p><p id="P12">The collected data was stored in the MGB Enterprise Data Warehouse (EDW). The challenge consisted of the extraction of ELHS SmartPhrases data stored in EDW as text and the transformation of this data into a structured dataset so that it could be uploaded to the external national registry of the ELHS. The real-life process and challenges with clinical data flow associated with patients visits at the epilepsy clinics are illustrated in <xref rid="F1" ref-type="fig">Fig. 1</xref>. By automating the process of clinical notes extraction and preprocessing &#x02013; as illustrated in <xref rid="F2" ref-type="fig">Fig. 2</xref> &#x02013; data analysis such as causal inferences and weekly QI dashboards can be potential applications.</p><p id="P13">Data was extracted from the hospital electronic medical record under a research protocol approved by the MGB Institutional Review Board; a waiver of informed consent was obtained. Clinical data were retrospectively analyzed for a cohort of adult patients (&#x02265; 18 years old). The cohort included 1806 patients seen at the Massachusetts General Hospital (MGH, n = 1371) and Brigham and Women&#x02019;s Hospital (BWH, n = 435) epilepsy clinics, between December 28th, 2018, to December 20th, 2021. Both hospitals use the MGB EPIC EHR software system. EHR data comprised clinical notes consisting of structured SmartPhrases (<xref rid="SD1" ref-type="supplementary-material">Supplementary Figure 1</xref>).</p></sec><sec id="S8"><label>2.2.</label><title>Data Collection and Processing</title><p id="P14">Data was acquired from MGB EDW using Microsoft Structured Query Language (SQL) Server Management Studio (SSMS). The methodology for notes preprocessing in order to obtain a structured dataset is presented in <xref rid="F2" ref-type="fig">Fig. 2</xref>. All notes from the patients seen at MGH and BWH epilepsy clinics were extracted. Patient identification (PatientID and medical record number (MRN)), healthcare provider, department identification, date of visit, note identification (NoteID), note line number (LineNBR) and the text notes were queried from the EDW databases and exported to an excel file. An example of a text note extracted from EDW for preprocessing is presented in <xref rid="SD1" ref-type="supplementary-material">Supplementary Figure 2</xref>. The EHR clinical notes were then processed using Python version 3.7. Notes were filtered to include only those pertaining to the structured ELHS SmartForms. These notes were sorted by NoteID and LineNBR, and then grouped by PatientID, MRN, DepartmentID, provider and visit date. Notes were then subjected to lowercasing, blank spaces were removed, and notes with less than one token (one word), were excluded. Once the notes were preprocessed, regular expressions (regexes) were applied to extract information regarding seizure control, namely if the patient has epilepsy, if the patient missed taking the ASM, if there are side effects from taking the ASM, or any barriers to adherence to the ASM, the seizure type according to the ILAE classification, the correspondent diagnostic certainty, date of last seizure since current visit and current seizure frequency. For each of these fields, a column was created with the correspondent categories assigned. Notes without categories assigned for at least one of the seizure control information fields, were removed.</p><p id="P15">The providers also documented one to four seizure types &#x02013; a, b, c and d &#x02013;, since a patient may have more than one type. Thus, for each seizure type, the provider selects the respective type, frequency, diagnostic certainty and date of last seizure. Usually, when a patient only has one type of seizure, the provider will only select the fields correspondent to seizure type &#x0201c;a&#x0201d;, while if the patient has two types of seizure, both seizure types &#x0201c;a&#x0201d; and &#x0201c;b&#x0201d; fields will be selected. And so on, until four types of seizure (a, b, c and d) are selected in case the patient has four seizure types.</p><p id="P16">We present examples of the regexes created, including the number of tokens captured for information extraction for each field in <xref rid="SD1" ref-type="supplementary-material">Supplementary Table 1</xref>. The number of tokens was set for each field based on the length of text of the corresponding categories. For example, for the field &#x0201c;Does the patient have epilepsy?&#x0201d; we captured the corresponding answer yes/no/unsure, with a total maximum length of 100 tokens (words). For special cases, including menus with multiple choices, the number of tokens was set to a maximum of 3000 since menus include multiple options. As an example, for the field &#x0201c;seizure type&#x0201d;, we have &#x0201c;Seizure type A ILAE CLASSIFICATION: Generalized [x] Tonic-clonic [] Absence typical [] Absence atypical [] Myoclonic absence [] Eyelid myoclonia [] Myoclonic [] Myoclonic atonic [] Myoclonic tonic [] Clonic [] Tonic Focal [] Focal aware without impairment of consciousness [] With observable motor components [] Involving subjective sensory or psychic phenomena only [] Focal with impairment of consciousness [] Focal evolving to bilateral convulsive seizure Unclassified [] Epileptic spasms (focal or generalized) [] Unclassified seizure type&#x0201d;. For this example, we captured all the seizure types displayed in the menu and the regex assigned the seizure type generalized tonic-clonic &#x0201c;GENTC&#x0201d;.</p><p id="P17">The regular expressions were designed in a flexible way. In case that a provider has indicated an actual calendar date for the field date of last seizure, instead of selecting one of the coded options (eg. 1WK &#x02013; one week ago or 6MON &#x02013; six months ago), the regex captured the date and assigned the correspondent category. This approach was also applied for the field &#x0201c;seizure frequency&#x0201d;. We provide an example with the text &#x0201c;Seizure type A current FREQUENCY since last visit with same provider: GTC - less than 2 years ago. Dejavu - unclear.&#x0201d;, in <xref rid="SD1" ref-type="supplementary-material">Supplementary Figure 3</xref> (case 1), where the regex assigned the corresponding category &#x0201c;YEAR&#x0201d; (less than once per year). We also analyzed cases where there was a complete note for the patient visit at the epilepsy clinic, however no information could be extracted, because the seizure control information fields consisted of &#x0201c; *** &#x0201d;, as presented in <xref rid="SD1" ref-type="supplementary-material">Supplementary Figure 3</xref> (case 2). For these few special cases, the SmartPhrase was not correctly saved by the system at the time of clinic visit. The regexes were refined iteratively by assessing notes with special cases and for each field where no category was assigned.</p></sec><sec id="S9"><label>2.3.</label><title>Performance evaluation</title><p id="P18">The authors (MF, SZ, CMM, MBW, LMVRM) manually reviewed 100 randomly selected notes (20 cases per author), for a total of 2000 fields (100 notes &#x000d7; 20 fields for seizure control metrics). The seizure control metrics included: whether the patient has epilepsy, whether the patient missed taking the ASM, if there are side effects from taking ASM(s), if there are barriers to adherence to the ASM, and for the four seizure types (a/b/c/d) the type, the corresponding diagnostic certainty, date of last seizure, and seizure frequency. We compared the performance of the regexes-based approach with the manual review results. Performance measures included precision, recall and F1 score to measure the regex-based approach compared to the gold standard of manual review. Precision is defined as the proportion of the seizure control metric items extracted by the regex-based approach which are true, recall is the proportion of true seizure control metrics items extracted by the algorithm and F1 score is the unweighted harmonic mean of precision and recall: (2 &#x000d7; precision &#x000d7; recall)/ (precision + recall).</p><p id="P19">The cohort population further analyzed included adult patients residing in the US seen at the epilepsy clinics, with information regarding their seizure type and corresponding frequency, present in the seizure control structured dataset.</p></sec></sec><sec id="S10"><label>3.</label><title>Results</title><sec id="S11"><label>3.1.</label><title>Summary of patient population</title><p id="P20">The EHR clinical notes were extracted from EDW and preprocessed. The categories for each seizure control information field were assigned according to the ELHS specifications, using regexes to obtain a structured dataset. There were 2125 patients with seizure control information in the dataset. After excluding patients younger than 18 years old, non-US residents and those who lacked both seizure type and frequency, we were left with 1806 patients (<xref rid="F3" ref-type="fig">Fig. 3</xref>).</p><p id="P21">The cohort of patients had a mean age of 43 years old (SD: 17), where 57% were female, 80% were white, and 84 % were non-Hispanic/Latino (<xref rid="T1" ref-type="table">Table 1</xref>). There were 5168 clinic visits logged by providers documenting both seizure type and frequency. During this period, providers documented 59 % patients having focal seizures, 35 % having generalized seizures and 6 % patients having other type. Of the cohort, 45 % patients had private insurance.</p></sec><sec id="S12"><label>3.2.</label><title>Performance results</title><p id="P22">From the manual review of 100 notes randomly selected, the experts identified 890 items in a total of 2000 fields (100 notes &#x000d7; 20 fields for seizure control metrics). The majority of notes were from patients with one type of epilepsy (seizure type a), with correspondent diagnostic certainty (diagnostic certainty a), date of last seizure (date of last seizure a), and seizure frequency (seizure frequency a). Thus, the majority of fields for the second, third and fourth types of epilepsy, b, c and d, respectively, were left blank, since these were not identified in the notes.</p><p id="P23">The regex-based approach captured 887 items from the 890 items identified by the experts, as presented in <xref rid="T2" ref-type="table">Table 2</xref>. The category &#x0201c;Not Applicable&#x0201d; was identified by experts for the metrics of seizure type, seizure frequency and date of last seizure. This category was not included in the ELHS specifications for these metrics, as depicted in <xref rid="SD1" ref-type="supplementary-material">Supplementary Table 2</xref>, thus it was not coded in the algorithm. Among 100 notes randomly selected, there were 3 false negative cases. The overall performance was approximately 100 % (precision: 1, recall: 0.996, F-score: 0.998). The performance obtained is mainly explained by the fact that notes are mostly structured, and might not reflect the overall performance, since it considers only 2 % (100/5168) of the notes in the study.</p></sec><sec id="S13"><label>3.3.</label><title>Analysis of documented seizure control metrics</title><p id="P24">The number of patients with documented fields in the notes can be depicted in <xref rid="F4" ref-type="fig">Fig. 4</xref>. Patients may have follow-up visits, thus the volume of notes is greater than the number of patients. For each patient, a category for a specific field may repeat (e.g. &#x0201c;YES&#x0201d; for the question &#x0201c;Does the patient have epilepsy?&#x0201d;) and for another field, such as seizure frequency, the category may vary for example, from &#x0201c;NEM&#x0201d; (at least once per year, but not every month) to &#x0201c;YEAR&#x0201d; (less than once per year).</p><p id="P25">From a total of 1806 patients with documented seizure type and frequency, almost all of the patients (N = 1804) had a category assigned for the field &#x0201c;Diagnostic certainty&#x0201d;, as well as for the field &#x0201c;Does the patient have epilepsy?&#x0201d; (N = 1801). The &#x0201c;Date of last seizure&#x0201d; field was also documented for the majority of patients (N = 1792). The fields less documented were the ones related to ASM, namely the fields: &#x0201c;In the past four weeks, how good of a job did the patient do at taking anti-seizure medicine?&#x0201d; (N = 1021), &#x0201c;Is the patient having any problems that are side effects of the medicine?&#x0201d; (N = 1018) and &#x0201c;Is the patient experiencing any barriers to adherence?&#x0201d; (N = 847). This is due to the fact that documentation of ASM was added in a later version of the SmartPhrase, and the first register date is July 24th, 2020. In a total of 1394 patients with notes prior to ASM documentation, approximately 73 % (N = 1021) have documentation of missed ASM, as observed in <xref rid="F4" ref-type="fig">Fig. 4 (a)</xref>.</p><p id="P26">We assessed the percentage of patients with reported seizure control metrics documented for the period before and after the implementation of ASM documentation, presented in <xref rid="F5" ref-type="fig">Fig. 5</xref>. For the most reported fields, the percentage of patients with reported seizure control metrics was consistently ~ 100 %. The fields with medication related information documented was approximately 73 % for both missed medication (N = 1021) and side effects (N = 1018) and 61 % for barriers to adherence (N = 847).</p><p id="P27">Finally, we assessed the distribution of categories assigned by providers per field in the study cohort (<xref rid="SD1" ref-type="supplementary-material">Supplementary Table 3</xref>). The majority of patients had epilepsy (N = 1554, 86 %) and was assigned Generalized tonic-clonic (N = 481), Focal with impairment of consciousness (N = 147) and Focal evolving to bilateral convulsive seizure (N = 33) for the three seizure types a, b and c, respectively. Since the number of patients with a fourth seizure type was reduced, we omitted the numbers for this seizure type. The majority of patients had a definite diagnostic &#x02013; summary of evidence suggesting 100 % confidence level &#x02013; for all three seizure types a (N = 1053), b (N = 344) and c (N = 65). For the majority, the date of last seizure types a, b and c (N = 504, N = 200 and N = 42) was more than 2 years prior to the visit; and the frequency since last visit for seizure types a, b and c (N = 561, N = 179 and N = 34) was less than once per year. For the fields related to ASM, when documented, the majority of patients did not miss more than two doses in a row of ASM (N = 590), did not present side effects to the ASM (N = 517) and did not experience any barriers to adherence (N = 492).</p></sec></sec><sec id="S14"><label>4.</label><title>Discussion</title><sec id="S15"><label>4.1.</label><title>Main findings</title><p id="P28">We collected standardized epilepsy care data in the EHR of patients seen at MGB. This data consisted of diagnosis of epilepsy, seizure type according to the ILAE classification, seizure frequency, date of last seizure and anti-seizure medication adherence and side effects. The data was extracted from EDW using structured query language and preprocessed using regexes. Using the ELHS data dictionary, we matched the fields in the preprocessed notes to organize the data into a structured format that we analyzed. We assessed the performance of our regex-based approach compared to experts&#x02019; manual review of 100 random notes. The overall performance of the regex-based approach was approximately 100 % across metrics, which we concluded was due to the fact that notes are mostly structured. We found that the majority of patients with documented seizure type and frequency had a definite diagnostic certainty of epilepsy as well as the date of last seizure. The majority of patients were also seizure free, with a date of last seizure of more than 2 years prior to the visit. As foreseen, the majority of patients in our cohort had epilepsy. Generalized tonic-clonic and focal with impairment of consciousness seizure types were the most predominant in our cohort. We concluded that the fields less documented were related to missed ASM, ASM side effects and barriers to adherence. This might be due to later introduction of ASM related fields, compared to the other metrics, which were documented since the beginning of SmartPhrase implementation. When documented, the majority of patients did not miss more than two doses in a row of ASM, did not present side effects to the ASM and did not experience any barriers to adherence. Being documentation of barriers to medication adherence one of the ELHS network-wide interventions, we reported these findings to providers to promote adherence to reporting. Therefore, we conclude that by using a text mining methodology applied to quality data, we can provide meaningful insights and engage neurologists to improve their patient outcomes. Thus, we are able to provide a tool that has the potential to improve the reporting of vital seizure control metrics, thereby improving patient care.</p></sec><sec id="S16"><label>4.2.</label><title>Prior work</title><p id="P29">Prior work has been done using QI methodologies applied to epilepsy research. In (<xref rid="R21" ref-type="bibr">Narayanan et al., 2017</xref>), the authors describe a methodology for building structured clinical documentation support tools in the EHR to support QI and define best practices in epilepsy. The authors describe how they incorporated these toolkits into their clinical workflow. They also shared the EHR tools developed with other epilepsy clinics as part of a Neurology Practice Based Research Network and applied these tools to conduct pragmatic trials using subgroup-based adaptive designs. A multidisciplinary QI team conducted analysis of data for prescribed seizure rescue medication doses to identify and improve inappropriately low dose prescriptions (<xref rid="R23" ref-type="bibr">Patel et al., 2020</xref>). The QI team identified areas of focus for improvement opportunities and developed the project objective based on the 2017 American Academy of Neurology (AAN) and Child Neurology Society (CNS) Quality Measure. The team created an automated monthly report to monitor prescribed seizure rescue medication dosing compliance. The team successfully decreased provider prescribed and signed under-dosed rescue medication orders by an average of 89 %. In (<xref rid="R7" ref-type="bibr">Cisneros-Franco et al., 2013</xref>), based on the AAN quality measures that should be observed at every patient visit, the authors compared the percentage of documentation of each measure before and after the implementation of a new worksheet in a third-level center. Documented measures included seizure type and frequency, etiology, electroencephalogram (EEG), magnetic resonance imaging/ computerized tomography (MRI/CT) head scans, ASM side effects, surgical therapy referral, safety counseling, preconception counseling-and physical exam. The authors concluded that the quality-oriented epilepsy worksheet led to a better practice standardization and documentation of AAN standards for diagnostic and counseling purposes.</p><p id="P30">In previous literature, methodologies have been presented to obtain seizure control metrics from clinical notes of patients seen at medical centers. A previous study (<xref rid="R3" ref-type="bibr">Barbour et al., 2019</xref>) used regexes to identify sudden unexpected death in epilepsy (SUDEP) risk factors in physician notes. Three variables related to SUDEP were considered in the study: generalized tonic-clonic seizure type, refractory epilepsy and potential or previous epilepsy surgery candidacy. Other metrics, such as seizure frequency, acknowledged by the authors as an important risk predictor, were not considered. The authors stated that there were limited NLP tools to reliably evaluate temporal characteristics, and none would be suitable for the analysis. Indeed, extracting key information from unstructured text is challenging. In our case, the SmartPhrase was designed to have specific categories associated with the seizure frequency, thus it was possible to reliable use regexes. The authors (<xref rid="R3" ref-type="bibr">Barbour et al., 2019</xref>) concluded that regexes were a feasible method to identify variables related to SUDEP risk and that their methods could be implemented to create large patient cohorts for research. <xref rid="R5" ref-type="bibr">Cui et al. (2012)</xref> developed the epilepsy data extraction and annotation (EpiDEA) rule-based system, which extracts information from epilepsy monitoring unit (EMU) discharge summaries. The study also focused on variables of interest for the study of SUDEP: seizure semiology, EEG and MRI patterns, and antiepileptic drug medication. The authors demonstrated the use of EpiDEA for cohort identification through use of an intuitive visual query interface that can be used by clinical researchers. <xref rid="R6" ref-type="bibr">Cui et al. (2014)</xref> also developed a rule-based information extraction system called Phenotype Extraction in Epilepsy (PEEP) to automatically identify epilepsy phenotypes with anatomical locations. The variables extracted by PEEP were epileptogenic zone, seizure semiology, lateralising sign, interictal and ictal EEG pattern. The authors demonstrated that their approach was effective in extracting complex epilepsy phenotypes for cohort identification, with an integrated ontology-driven visual query interface. <xref rid="R27" ref-type="bibr">Sullivan et al. (2014)</xref> used a machine learning based NLP pipeline to identify a rare epilepsy syndrome from discharge summaries and EEG reports. The authors concluded that their model could assist physicians to identify the epilepsy syndrome not previously diagnosed, potentially improving seizure control and quality of life for the patients. <xref rid="R13" ref-type="bibr">Fonferko-Shadrach et al. (2019)</xref> developed the extraction of epilepsy clinical text (ExECT) system, combining rule-based and statistical techniques to extract information from epilepsy clinic letters. Nine variables were considered: epilepsy diagnosis, epilepsy type, focal seizures, generalized seizures, seizure frequency, medication name (identified when accompanied by a quantity and frequency), CT, MRI and EEG. The authors concluded that their methods could enhance routinely collected data for research and be used in clinical practice to record patient information in a structured manner.</p><p id="P31">We did not proceed with a comparison between the performance obtained in our study with that of other studies. We developed the NLP approach for semi-structured data while other studies developed methodologies for unstructured text either from clinical letters or discharge summaries, which is more challenging. We concluded that applying our regexes-based approach to semi-structured text enabled us to capture a higher number of seizure control metrics more reliably.</p></sec><sec id="S17"><label>4.3.</label><title>Limitations</title><p id="P32">The DI methodology was developed for specific SmartPhrases in two academic medical centers located in the same geographic region (Boston, Massachusetts), both of which use the EPIC EHR and may not be representative of other US and non-US populations limiting the generalizability of our methodology across populations and hospital settings. Since the methodology still assumes knowledge and granted access to the software, future work includes providing any user, neurologists and quality improvers, with an internet-based notebook to easily obtain the preprocessed seizure control information data. Nonetheless, our methodology presents ease of replicability, it is not time costly, and it is free of charge.</p></sec><sec id="S18"><label>4.4.</label><title>Conclusions</title><p id="P33">We designed a methodology to improve the quality of reporting key seizure control metrics. We implemented the methodology that captures seizure control information on a weekly basis to be available to neurologists. We concluded that structured data can be feasibly extracted from text notes of epilepsy patients for weekly reporting to a national learning healthcare system. Finally, translating the reported statistics to improvements in standard of care and outcomes for the patients integrated in the ELHS project is needed and proposed as future work.</p></sec></sec><sec sec-type="supplementary-material" id="SM1"><title>Supplementary Material</title><supplementary-material id="SD1" position="float" content-type="local-data"><label>supplement</label><media xlink:href="NIHMS1861737-supplement-supplement.docx" id="d64e520" position="anchor"/></supplementary-material></sec></body><back><ack id="S19"><title>Acknowledgements</title><p id="P34">Christopher M. McGraw was supported by the National Institutes of Health (NIH-NINDS K08NS118107) and reports no conflict of interest. Lidia M. V. R. Moura was supported by the Centers for Diseases Control and Prevention (U48DP006377), the National Institutes of Health (NIH-NIA 5K08AG053380-02, NIH-NIA 5R01AG062282-02, NIH-NIA 2P01AG032952-11, NIH-NIA 3R01AG062282-03S1, NIH-NIA 1R01AG073410 - 01), and the Epilepsy Foundation of America and reports no conflict of interest.</p><p id="P35">Claire Jacobs received support from Harvard Catalyst | The Harvard Clinical and Translational Science Center (National Center for Advancing Translational Sciences, National Institutes of Health Award UL1 TR002541) and financial contributions from Harvard University and its affiliated academic healthcare centers and reports no conflicts of interest.</p><p id="P36">M. Brandon Westover was supported by the Glenn Foundation for Medical Research and American Federation for Aging Research (Breakthroughs in Gerontology Grant); American Academy of Sleep Medicine (AASM Foundation Strategic Research Award); Football Players Health Study (FPHS) at Harvard University; Department of Defense through a subcontract from Moberg ICU Solutions, Inc; by the NIH (1R01NS102190, 1R01NS102574, 1R01NS107291, 1RF1AG064312). M. Brandon Westover is a co-founder of Beacon Biosignals and reports no conflict of interest.</p></ack><fn-group><fn fn-type="COI-statement" id="FN2"><p id="P37">Declarations of Interest</p><p id="P38">None.</p></fn><fn id="FN3"><p id="P39">Appendix A. Supporting information</p><p id="P40">Supplementary data associated with this article can be found in the online version at <ext-link xlink:href="10.1016/j.eplepsyres.2022.107013" ext-link-type="doi">doi:10.1016/j.eplepsyres.2022.107013</ext-link>.</p></fn></fn-group><glossary><title>Abbreviations:</title><def-list><def-item><term>ASM</term><def><p id="P41">Anti-Seizure Medication</p></def></def-item><def-item><term>BWH</term><def><p id="P42">Brigham and Women&#x02019;s Hospital</p></def></def-item><def-item><term>CDE</term><def><p id="P43">Common Data Elements</p></def></def-item><def-item><term>CRFs</term><def><p id="P44">Case Report Forms</p></def></def-item><def-item><term>DI</term><def><p id="P45">Data Integration</p></def></def-item><def-item><term>EDW</term><def><p id="P46">Enterprise Data Warehouse</p></def></def-item><def-item><term>EHR</term><def><p id="P47">Electronic Health Record</p></def></def-item><def-item><term>ELHS</term><def><p id="P48">Epilepsy Learning Healthcare System</p></def></def-item><def-item><term>ILAE</term><def><p id="P49">International League Against Epilepsy</p></def></def-item><def-item><term>LHS</term><def><p id="P50">Learning Healthcare Systems</p></def></def-item><def-item><term>LineNBR</term><def><p id="P51">note line number</p></def></def-item><def-item><term>MGB</term><def><p id="P52">Mass General Brigham</p></def></def-item><def-item><term>MRN</term><def><p id="P53">Medical Record Number</p></def></def-item><def-item><term>NLP</term><def><p id="P54">Natural language processing</p></def></def-item><def-item><term>NoteID</term><def><p id="P55">Note Identification</p></def></def-item><def-item><term>PatientID</term><def><p id="P56">Patient Identification</p></def></def-item><def-item><term>PROMs</term><def><p id="P57">Patient-Reported Outcome Measures</p></def></def-item><def-item><term>QI</term><def><p id="P58">Quality Improvement</p></def></def-item><def-item><term>Regex</term><def><p id="P59">regular expression</p></def></def-item><def-item><term>SQL</term><def><p id="P60">Structured Query Language</p></def></def-item><def-item><term>SSMS</term><def><p id="P61">Microsoft SQL Server Management Studio</p></def></def-item></def-list></glossary><ref-list><title>References</title><ref id="R1"><mixed-citation publication-type="journal"><name><surname>Bell</surname><given-names>D</given-names></name>, <name><surname>Gachuhi</surname><given-names>N</given-names></name>, <name><surname>Assefi</surname><given-names>N</given-names></name>, <year>2018</year>. <article-title>Dynamic clinical algorithms: digital technology can transform health care decision-making</article-title>. <source>Am. J. Trop. Med. Hyg</source>
<volume>98</volume> (<issue>1</issue>), <fpage>9</fpage>&#x02013;<lpage>14</lpage>. <pub-id pub-id-type="doi">10.4269/ajtmh.17-0477</pub-id>.<pub-id pub-id-type="pmid">29141738</pub-id></mixed-citation></ref><ref id="R2"><mixed-citation publication-type="journal"><name><surname>Bindman</surname><given-names>AB</given-names></name>, <name><surname>Pronovost</surname><given-names>PJ</given-names></name>, <name><surname>Asch</surname><given-names>DA</given-names></name>, <year>2018</year>. <article-title>Funding innovation in a learning health care system</article-title>. <source>JAMA</source>
<volume>319</volume> (<issue>2</issue>), <fpage>119</fpage>&#x02013;<lpage>120</lpage>. <pub-id pub-id-type="doi">10.1001/jama.2017.18205</pub-id>.<pub-id pub-id-type="pmid">29270611</pub-id></mixed-citation></ref><ref id="R3"><mixed-citation publication-type="journal"><name><surname>Barbour</surname><given-names>K</given-names></name>, <name><surname>Hesdorffer</surname><given-names>DC</given-names></name>, <name><surname>Tian</surname><given-names>N</given-names></name>, <name><surname>Yozawitz</surname><given-names>EG</given-names></name>, <name><surname>McGoldrick</surname><given-names>PE</given-names></name>, <name><surname>Wolf</surname><given-names>S</given-names></name>, <name><surname>McDonough</surname><given-names>TL</given-names></name>, <name><surname>Nelson</surname><given-names>A</given-names></name>, <name><surname>Loddenkemper</surname><given-names>T</given-names></name>, <name><surname>Basma</surname><given-names>N</given-names></name>, <name><surname>Johnson</surname><given-names>SB</given-names></name>, <name><surname>Grinspan</surname><given-names>Z</given-names></name>, <year>2019</year>. <article-title>Automated detection of sudden unexpected death in epilepsy risk factors in electronic medical records using natural language processing</article-title>. <source>Epilepsia</source>
<volume>60</volume> (<issue>6</issue>), <fpage>1209</fpage>&#x02013;<lpage>1220</lpage>. <pub-id pub-id-type="doi">10.1111/epi.15966</pub-id>.<pub-id pub-id-type="pmid">31111463</pub-id></mixed-citation></ref><ref id="R4"><mixed-citation publication-type="journal"><name><surname>Budrionis</surname><given-names>A</given-names></name>, <name><surname>Bellika</surname><given-names>JG</given-names></name>, <year>2016</year>. <article-title>The learning healthcare system: where are we now? a systematic review</article-title>. <source>J. Biomed. Inform</source>
<volume>64</volume>, <fpage>87</fpage>&#x02013;<lpage>92</lpage>. <pub-id pub-id-type="doi">10.1016/j.jbi.2016.09.018</pub-id>.<pub-id pub-id-type="pmid">27693565</pub-id></mixed-citation></ref><ref id="R5"><mixed-citation publication-type="journal"><name><surname>Cui</surname><given-names>L</given-names></name>, <name><surname>Bozorgi</surname><given-names>A</given-names></name>, <name><surname>Lhatoo</surname><given-names>SD</given-names></name>, <name><surname>Zhang</surname><given-names>GQ</given-names></name>, <name><surname>Sahoo</surname><given-names>SS</given-names></name>, <year>2012</year>. <article-title>EpiDEA: extracting structured epilepsy and seizure information from patient discharge summaries for cohort identification</article-title>. <source>AMIA Annu Symp Proc</source>. <volume>2012</volume>, <fpage>1191</fpage>&#x02013;<lpage>1200</lpage>.<pub-id pub-id-type="pmid">23304396</pub-id></mixed-citation></ref><ref id="R6"><mixed-citation publication-type="journal"><name><surname>Cui</surname><given-names>L</given-names></name>, <name><surname>Sahoo</surname><given-names>SS</given-names></name>, <name><surname>Lhatoo</surname><given-names>SD</given-names></name>, <name><surname>Garg</surname><given-names>G</given-names></name>, <name><surname>Rai</surname><given-names>P</given-names></name>, <name><surname>Bozorgi</surname><given-names>A</given-names></name>, <name><surname>Zhang</surname><given-names>GQ</given-names></name>, <year>2014</year>. <article-title>Complex epilepsy phenotype extraction from narrative clinical discharge summaries</article-title>. <source>J. Biomed Inform</source>
<volume>51</volume>, <fpage>272</fpage>&#x02013;<lpage>279</lpage>. <pub-id pub-id-type="doi">10.1016/j.jbi.2014.06.006</pub-id>.<pub-id pub-id-type="pmid">24973735</pub-id></mixed-citation></ref><ref id="R7"><mixed-citation publication-type="journal"><name><surname>Cisneros-Franco</surname><given-names>JM</given-names></name>, <name><surname>D&#x000ed;az-Torres</surname><given-names>MA</given-names></name>, <name><surname>Rodr&#x000ed;guez-Casta&#x000f1;eda</surname><given-names>JB</given-names></name>, <name><surname>Mart&#x000ed;nez-Silva</surname><given-names>A</given-names></name>, <name><surname>Gutierrez-Herrera</surname><given-names>MA</given-names></name>, <name><surname>San-Juan</surname><given-names>D</given-names></name>, <year>2013</year>. <article-title>Impact of the implementation of the AAN epilepsy quality measures on the medical records in a university hospital</article-title>. <source>BMC Neurol</source>. <volume>13</volume>, <fpage>112</fpage>. <pub-id pub-id-type="doi">10.1186/1471-2377-13-112</pub-id>.<pub-id pub-id-type="pmid">23984949</pub-id></mixed-citation></ref><ref id="R8"><mixed-citation publication-type="journal"><name><surname>Donahue</surname><given-names>MA</given-names></name>, <name><surname>Herman</surname><given-names>ST</given-names></name>, <name><surname>Dass</surname><given-names>D</given-names></name>, <name><surname>Farrell</surname><given-names>K</given-names></name>, <name><surname>Kukla</surname><given-names>A</given-names></name>, <name><surname>Abend</surname><given-names>NS</given-names></name>, <name><surname>Moura</surname><given-names>LMVR</given-names></name>, <name><surname>Buchhalter</surname><given-names>JR</given-names></name>, <name><surname>Fureman</surname><given-names>BE</given-names></name>, <year>2021</year>. <article-title>Establishing a learning healthcare system to improve health outcomes for people with epilepsy</article-title>. <source>Epilepsy Behav.: EB</source>
<volume>117</volume>, <fpage>107805</fpage>. <pub-id pub-id-type="doi">10.1016/j.yebeh.2021.107805</pub-id>.</mixed-citation></ref><ref id="R9"><mixed-citation publication-type="journal"><name><surname>Elbattah</surname><given-names>M</given-names></name>, <name><surname>Arnaud</surname><given-names>E</given-names></name>, <name><surname>Gignon</surname><given-names>M</given-names></name>, <name><surname>Dequen</surname><given-names>G</given-names></name>, <year>2021</year>. <article-title>The role of text analytics in healthcare: a review of recent developments and applications</article-title>. <source>HEALTHINF</source>
<fpage>825</fpage>&#x02013;<lpage>832</lpage>. <pub-id pub-id-type="doi">10.5220/0010414508250832</pub-id>.</mixed-citation></ref><ref id="R10"><mixed-citation publication-type="journal"><name><surname>Enticott</surname><given-names>J</given-names></name>, <name><surname>Johnson</surname><given-names>A</given-names></name>, <name><surname>Teede</surname><given-names>H</given-names></name>, <year>2021</year>. <article-title>Learning health systems using data to drive healthcare improvement and impact: a systematic review</article-title>. <source>BMC Health Serv. Res</source>
<volume>21</volume> (<issue>1</issue>), <fpage>200</fpage>. <pub-id pub-id-type="doi">10.1186/s12913-021-06215-8</pub-id>.<pub-id pub-id-type="pmid">33663508</pub-id></mixed-citation></ref><ref id="R11"><mixed-citation publication-type="webpage"><collab>Epilepsy Foundation</collab>(<year>2020b</year>). <source>What is Epilepsy? Disease or Disorder?</source>
<comment><ext-link xlink:href="https://www.epilepsy.com/learn/about-epilepsy-basics/what-epilepsy" ext-link-type="uri">https://www.epilepsy.com/learn/about-epilepsy-basics/what-epilepsy</ext-link></comment> (<comment>Accessed</comment>
<date-in-citation>11 December 2021</date-in-citation>).</mixed-citation></ref><ref id="R12"><mixed-citation publication-type="webpage"><collab>Epilepsy Foundation</collab>, <year>2020a</year>. <source>Seizure Control</source>
<comment><ext-link xlink:href="https://www.epilepsy.com/aimforzero/seizure-control" ext-link-type="uri">https://www.epilepsy.com/aimforzero/seizure-control</ext-link></comment> (<comment>Accessed</comment>
<date-in-citation>15 December 2021</date-in-citation>).</mixed-citation></ref><ref id="R13"><mixed-citation publication-type="journal"><name><surname>Fonferko-Shadrach</surname><given-names>B</given-names></name>, <name><surname>Lacey</surname><given-names>AS</given-names></name>, <name><surname>Roberts</surname><given-names>A</given-names></name>, <name><surname>Akbari</surname><given-names>A</given-names></name>, <name><surname>Thompson</surname><given-names>S</given-names></name>, <name><surname>Ford</surname><given-names>DV</given-names></name>, <name><surname>Lyons</surname><given-names>RA</given-names></name>, <name><surname>Rees</surname><given-names>MI</given-names></name>, <name><surname>Pickrell</surname><given-names>WO</given-names></name>, <year>2019</year>. <article-title>Using natural language processing to extract structured epilepsy data from unstructured clinic letters: development and validation of the ExECT (extraction of epilepsy clinical text) system</article-title>. <source>BMJ Open</source>. <volume>9</volume> (<issue>4</issue>), <fpage>e023232</fpage>
<pub-id pub-id-type="doi">10.1136/bmjopen-2018-023232</pub-id>.</mixed-citation></ref><ref id="R14"><mixed-citation publication-type="journal"><name><surname>Forsgren</surname><given-names>L</given-names></name>, <name><surname>Beghi</surname><given-names>E</given-names></name>, <name><surname>Oun</surname><given-names>A</given-names></name>, <name><surname>Sillanp&#x000e4;&#x000e4;</surname><given-names>M</given-names></name>, <year>2005</year>. <article-title>The epidemiology of epilepsy in Europe - a systematic review</article-title>. <source>Eur. J. Neurol</source>
<volume>12</volume> (<issue>4</issue>), <fpage>245</fpage>&#x02013;<lpage>253</lpage>. <pub-id pub-id-type="doi">10.1111/j.1468-1331.2004.00992.x</pub-id>.<pub-id pub-id-type="pmid">15804240</pub-id></mixed-citation></ref><ref id="R15"><mixed-citation publication-type="journal"><name><surname>Loring</surname><given-names>DW</given-names></name>, <name><surname>Lowenstein</surname><given-names>DH</given-names></name>, <name><surname>Barbaro</surname><given-names>NM</given-names></name>, <name><surname>Fureman</surname><given-names>BE</given-names></name>, <name><surname>Odenkirchen</surname><given-names>J</given-names></name>, <name><surname>Jacobs</surname><given-names>MP</given-names></name>, <name><surname>Austin</surname><given-names>JK</given-names></name>, <name><surname>Dlugos</surname><given-names>DJ</given-names></name>, <name><surname>French</surname><given-names>JA</given-names></name>, <name><surname>Gaillard</surname><given-names>WD</given-names></name>, <name><surname>Hermann</surname><given-names>BP</given-names></name>, <name><surname>Hesdorffer</surname><given-names>DC</given-names></name>, <name><surname>Roper</surname><given-names>SN</given-names></name>, <name><surname>Van Cott</surname><given-names>AC</given-names></name>, <name><surname>Grinnon</surname><given-names>S</given-names></name>, <name><surname>Stout</surname><given-names>A</given-names></name>, <year>2011</year>. <article-title>Common data elements in epilepsy research: development and implementation of the NINDS epilepsy CDE project</article-title>. <source>Epilepsia</source>
<volume>52</volume> (<issue>6</issue>), <fpage>1186</fpage>&#x02013;<lpage>1191</lpage>. <pub-id pub-id-type="doi">10.1111/j.1528-1167.2011.03018.x</pub-id>.<pub-id pub-id-type="pmid">21426327</pub-id></mixed-citation></ref><ref id="R16"><mixed-citation publication-type="journal"><name><surname>Margolis</surname><given-names>PA</given-names></name>, <name><surname>Peterson</surname><given-names>LE</given-names></name>, <name><surname>Seid</surname><given-names>M</given-names></name>, <year>2013</year>. <article-title>Collaborative chronic care networks (C3Ns) to transform chronic illness care</article-title>. <source>Pediatrics</source>
<volume>131</volume> (<issue>Suppl 4</issue>), <fpage>S219</fpage>&#x02013;<lpage>S223</lpage>. <pub-id pub-id-type="doi">10.1542/peds.2012-3786J</pub-id>.<pub-id pub-id-type="pmid">23729764</pub-id></mixed-citation></ref><ref id="R17"><mixed-citation publication-type="journal"><name><surname>McLachlan</surname><given-names>S</given-names></name>, <name><surname>Potts</surname><given-names>HWW</given-names></name>, <name><surname>Dube</surname><given-names>K</given-names></name>, <name><surname>Buchanan</surname><given-names>D</given-names></name>, <name><surname>Lean</surname><given-names>S</given-names></name>, <name><surname>Gallagher</surname><given-names>T</given-names></name>, <name><surname>Johnson</surname><given-names>O</given-names></name>, <name><surname>Daley</surname><given-names>B</given-names></name>, <name><surname>Marsh</surname><given-names>W</given-names></name>, <name><surname>Fenton</surname><given-names>N</given-names></name>, <year>2018</year>. <article-title>The heimdall framework for supporting characterisation of learning health systems</article-title>. <source>J. Innov. Health Inform</source>
<volume>25</volume> (<issue>2</issue>), <fpage>77</fpage>&#x02013;<lpage>87</lpage>. <pub-id pub-id-type="doi">10.14236/jhi.v25i2.996</pub-id>.<pub-id pub-id-type="pmid">30398449</pub-id></mixed-citation></ref><ref id="R18"><mixed-citation publication-type="journal"><name><surname>Menear</surname><given-names>M</given-names></name>, <name><surname>Blanchette</surname><given-names>M-A</given-names></name>, <name><surname>Demers-Payette</surname><given-names>O</given-names></name>, <name><surname>Roy</surname><given-names>D</given-names></name>, <year>2019</year>. <article-title>A framework for value-creating learning health systems</article-title>. <source>Health Res. Policy Syst</source>
<volume>17</volume> (<issue>1</issue>), <fpage>79</fpage>. <pub-id pub-id-type="doi">10.1186/s12961-019-0477-3</pub-id>.<pub-id pub-id-type="pmid">31399114</pub-id></mixed-citation></ref><ref id="R19"><mixed-citation publication-type="journal"><name><surname>Mesk&#x000f3;</surname><given-names>B</given-names></name>, <name><surname>Drobni</surname><given-names>Z</given-names></name>, <name><surname>B&#x000e9;nyei</surname><given-names>&#x000c9;</given-names></name>, <name><surname>Gergely</surname><given-names>B</given-names></name>, <name><surname>Gy&#x00151;rffy</surname><given-names>Z</given-names></name>, <year>2017</year>. <article-title>Digital health is a cultural transformation of traditional healthcare</article-title>. <source>MHealth</source>
<volume>3</volume>, <fpage>38</fpage>. <pub-id pub-id-type="doi">10.21037/mhealth.2017.08.07</pub-id>.<pub-id pub-id-type="pmid">29184890</pub-id></mixed-citation></ref><ref id="R20"><mixed-citation publication-type="journal"><name><surname>Morain</surname><given-names>SR</given-names></name>, <name><surname>Kass</surname><given-names>NE</given-names></name>, <name><surname>Grossmann</surname><given-names>C</given-names></name>, <year>2017</year>. <article-title>What allows a health care system to become a learning health care system: results from interviews with health system leaders</article-title>. <source>Learn. Health Syst</source>
<volume>1</volume> (<issue>1</issue>), <fpage>e10015</fpage>
<pub-id pub-id-type="doi">10.1002/lrh2.10015</pub-id>.<pub-id pub-id-type="pmid">31245552</pub-id></mixed-citation></ref><ref id="R21"><mixed-citation publication-type="journal"><name><surname>Narayanan</surname><given-names>J</given-names></name>, <name><surname>Dobrin</surname><given-names>S</given-names></name>, <name><surname>Choi</surname><given-names>J</given-names></name>, <name><surname>Rubin</surname><given-names>S</given-names></name>, <name><surname>Pham</surname><given-names>A</given-names></name>, <name><surname>Patel</surname><given-names>V</given-names></name>, <name><surname>Frigerio</surname><given-names>R</given-names></name>, <name><surname>Maurer</surname><given-names>D</given-names></name>, <name><surname>Gupta</surname><given-names>P</given-names></name>, <name><surname>Link</surname><given-names>L</given-names></name>, <name><surname>Walters</surname><given-names>S</given-names></name>, <name><surname>Wang</surname><given-names>C</given-names></name>, <name><surname>Ji</surname><given-names>Y</given-names></name>, <name><surname>Maraganore</surname><given-names>DM</given-names></name>, <year>2017</year>. <article-title>Structured clinical documentation in the electronic medical record to improve quality and to support practice-based research in epilepsy</article-title>. <source>Epilepsia</source>
<volume>58</volume> (<issue>1</issue>), <fpage>68</fpage>&#x02013;<lpage>76</lpage>. <pub-id pub-id-type="doi">10.1111/epi.13607</pub-id>.</mixed-citation></ref><ref id="R22"><mixed-citation publication-type="journal"><name><surname>Ngugi</surname><given-names>AK</given-names></name>, <name><surname>Bottomley</surname><given-names>C</given-names></name>, <name><surname>Kleinschmidt</surname><given-names>I</given-names></name>, <name><surname>Sander</surname><given-names>JW</given-names></name>, <name><surname>Newton</surname><given-names>CR</given-names></name>, <year>2010</year>. <article-title>Estimation of the burden of active and life-time epilepsy: a meta-analytic approach</article-title>. <source>Epilepsia</source>
<volume>51</volume> (<issue>5</issue>), <fpage>883</fpage>&#x02013;<lpage>890</lpage>. <pub-id pub-id-type="doi">10.1111/j.1528-1167.2009.02481.x</pub-id>.<pub-id pub-id-type="pmid">20067507</pub-id></mixed-citation></ref><ref id="R23"><mixed-citation publication-type="journal"><name><surname>Patel</surname><given-names>AD</given-names></name>, <name><surname>Herbst</surname><given-names>J</given-names></name>, <name><surname>Gibson</surname><given-names>A</given-names></name>, <name><surname>Karn</surname><given-names>M</given-names></name>, <name><surname>Terry</surname><given-names>D</given-names></name>, <name><surname>Debs</surname><given-names>A</given-names></name>, <name><surname>Yarosz</surname><given-names>S</given-names></name>, <name><surname>Parker</surname><given-names>W</given-names></name>, <name><surname>Cohen</surname><given-names>DM</given-names></name>, <year>2020</year>. <article-title>Using quality improvement to implement the CNS/AAN quality measure on rescue medication for seizures</article-title>. <source>Epilepsia</source>
<volume>61</volume> (<issue>12</issue>), <fpage>2712</fpage>&#x02013;<lpage>2719</lpage>. <pub-id pub-id-type="doi">10.1111/epi.16713</pub-id>.<pub-id pub-id-type="pmid">33063879</pub-id></mixed-citation></ref><ref id="R24"><mixed-citation publication-type="book"><name><surname>Sarkar</surname><given-names>D</given-names></name>, <year>2019</year>. <source>Text Analytics with Python: A Practitioner&#x02019;s Guide to Natural Language Processing</source>. <publisher-name>Apress</publisher-name>, <publisher-loc>Bangalore</publisher-loc>.</mixed-citation></ref><ref id="R25"><mixed-citation publication-type="journal"><name><surname>Scheffer</surname><given-names>IE</given-names></name>, <name><surname>Berkovic</surname><given-names>S</given-names></name>, <name><surname>Capovilla</surname><given-names>G</given-names></name>, <name><surname>Connolly</surname><given-names>MB</given-names></name>, <name><surname>French</surname><given-names>J</given-names></name>, <name><surname>Guilhoto</surname><given-names>L</given-names></name>, <name><surname>Hirsch</surname><given-names>E</given-names></name>, <name><surname>Jain</surname><given-names>S</given-names></name>, <name><surname>Mathern</surname><given-names>GW</given-names></name>, <name><surname>Mosh&#x000e9;</surname><given-names>SL</given-names></name>, <name><surname>Nordli</surname><given-names>DR</given-names></name>, <name><surname>Perucca</surname><given-names>E</given-names></name>, <name><surname>Tomson</surname><given-names>T</given-names></name>, <name><surname>Wiebe</surname><given-names>S</given-names></name>, <name><surname>Zhang</surname><given-names>Y-H</given-names></name>, <name><surname>Zuberi</surname><given-names>SM</given-names></name>, <year>2017</year>. <article-title>ILAE classification of the epilepsies: position paper of the ILAE commission for classification and terminology</article-title>. <source>Epilepsia</source>
<volume>58</volume> (<issue>4</issue>), <fpage>512</fpage>&#x02013;<lpage>521</lpage>. <pub-id pub-id-type="doi">10.1111/epi.13709</pub-id>.<pub-id pub-id-type="pmid">28276062</pub-id></mixed-citation></ref><ref id="R26"><mixed-citation publication-type="journal"><name><surname>Scobie</surname><given-names>S</given-names></name>, <name><surname>Castle-Clarke</surname><given-names>S</given-names></name>, <year>2019</year>. <article-title>Implementing learning health systems in the UK NHS: policy actions to improve collaboration and transparency and support innovation and better use of analytics</article-title>. <source>Learn. Health Syst</source>
<volume>4</volume> (<issue>1</issue>), <fpage>e10209</fpage>
<pub-id pub-id-type="doi">10.1002/lrh2.10209</pub-id>.<pub-id pub-id-type="pmid">31989031</pub-id></mixed-citation></ref><ref id="R27"><mixed-citation publication-type="journal"><name><surname>Sullivan</surname><given-names>R</given-names></name>, <name><surname>Yao</surname><given-names>R</given-names></name>, <name><surname>Jarrar</surname><given-names>R</given-names></name>, <name><surname>Buchhalter</surname><given-names>J</given-names></name>, <name><surname>Gonzalez</surname><given-names>G</given-names></name>, <year>2014</year>. <article-title>Text Classification towards Detecting Misdiagnosis of an Epilepsy Syndrome in a Pediatric Population</article-title>. <source>AMIA Annu Symp Proc</source>. <fpage>1082</fpage>&#x02013;<lpage>1087</lpage>.<pub-id pub-id-type="pmid">25954418</pub-id></mixed-citation></ref><ref id="R28"><mixed-citation publication-type="journal"><name><surname>Teede</surname><given-names>HJ</given-names></name>, <name><surname>Johnson</surname><given-names>A</given-names></name>, <name><surname>Buttery</surname><given-names>J</given-names></name>, <name><surname>Jones</surname><given-names>CA</given-names></name>, <name><surname>Boyle</surname><given-names>DI</given-names></name>, <name><surname>Jennings</surname><given-names>GL</given-names></name>, <name><surname>Shaw</surname><given-names>T</given-names></name>, <year>2019</year>. <article-title>Australian health research alliance: national priorities in data-driven health care improvement</article-title>. <source>Med. J. Aust</source>
<volume>211</volume> (<issue>11</issue>), <fpage>494</fpage>&#x02013;<lpage>497</lpage>. <pub-id pub-id-type="doi">10.5694/mja2.50409</pub-id>.<pub-id pub-id-type="pmid">31733072</pub-id></mixed-citation></ref><ref id="R29"><mixed-citation publication-type="journal"><name><surname>Thimbleby</surname><given-names>H</given-names></name>, <year>2013</year>. <article-title>Technology and the future of healthcare</article-title>. <source>J. Public Health Res</source>
<volume>2</volume> (<issue>3</issue>), <fpage>e28</fpage>
<pub-id pub-id-type="doi">10.4081/jphr.2013.e28</pub-id>.<pub-id pub-id-type="pmid">25170499</pub-id></mixed-citation></ref><ref id="R30"><mixed-citation publication-type="webpage"><collab>World Health Organization</collab> (<year>2019a</year>). <source>Epilepsy</source>. <comment><ext-link xlink:href="https://www.who.int/news-room/fact-sheets/detail/epilepsy" ext-link-type="uri">https://www.who.int/news-room/fact-sheets/detail/epilepsy</ext-link></comment> (<comment>Accessed</comment>
<date-in-citation>29 December 2021</date-in-citation>).</mixed-citation></ref><ref id="R31"><mixed-citation publication-type="journal"><collab>World Health Organization</collab> (<year>2019b</year>). <source>Health Service Data&#x02014;WHO</source>. <comment><ext-link xlink:href="https://www.who.int/data/data-collection-tools/health-service-data" ext-link-type="uri">https://www.who.int/data/data-collection-tools/health-service-data</ext-link></comment> (<comment>Accessed</comment>
<date-in-citation>15 December 2021</date-in-citation>).</mixed-citation></ref></ref-list></back><floats-group><fig position="float" id="F1"><label>Fig. 1.</label><caption><p id="P62">Clinical data flow process. EDW &#x02013; Enterprise Data Warehouse. EHR &#x02013; Electronic Health Record. QI &#x02013; Quality Improvement. PROMs &#x02013; Patient-Reported Outcome Measures.</p></caption><graphic xlink:href="nihms-1861737-f0001" position="float"/></fig><fig position="float" id="F2"><label>Fig. 2.</label><caption><p id="P63">Methodology for notes preprocessing. EDW &#x02013; Enterprise Data Warehouse. EHR &#x02013; Electronic Health Record. MRN &#x02013; Medical Record Number. ELHS &#x02013; The Epilepsy Learning Healthcare System. ILAE &#x02013; The International League Against Epilepsy. ASM &#x02013; Anti-seizure medication. ELHS specifications categories and respective acronyms are presented in <xref rid="SD1" ref-type="supplementary-material">Supplementary Table 2</xref>.</p></caption><graphic xlink:href="nihms-1861737-f0002" position="float"/></fig><fig position="float" id="F3"><label>Fig. 3.</label><caption><p id="P64">Inclusion and exclusion criteria to select the cohort population.</p></caption><graphic xlink:href="nihms-1861737-f0003" position="float"/></fig><fig position="float" id="F4"><label>Fig. 4.</label><caption><p id="P65">Statistics for all notes containing seizure control metrics captured with the regular expressions for the study cohort population. The number of notes and patients are displayed for (a) the period before and after the implementation of ASM documentation, and (b) the study period. ILAE &#x02013; The International League Against Epilepsy. ASM &#x02013; Anti-seizure medication.</p></caption><graphic xlink:href="nihms-1861737-f0004" position="float"/></fig><fig position="float" id="F5"><label>Fig. 5.</label><caption><p id="P66">Percentage of patients in the study cohort population with documented seizure control metrics before and after ASM documentation date. ILAE &#x02013; The International League Against Epilepsy. ASM &#x02013; Anti-seizure medication.</p></caption><graphic xlink:href="nihms-1861737-f0005" position="float"/></fig><table-wrap position="float" id="T1"><label>Table 1</label><caption><p id="P67">Characteristics of the study cohort population.</p></caption><table frame="hsides" rules="groups"><colgroup span="1"><col align="left" valign="middle" span="1"/><col align="left" valign="middle" span="1"/></colgroup><thead><tr><th align="left" valign="top" rowspan="1" colspan="1">Characteristic</th><th align="left" valign="top" rowspan="1" colspan="1">Study cohort (n = 1806)</th></tr></thead><tbody><tr><td align="left" valign="top" rowspan="1" colspan="1">Number of clinic visits<sup><xref rid="TFN3" ref-type="table-fn">(1)</xref></sup></td><td align="left" valign="top" rowspan="1" colspan="1">5168</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Age (years, mean (SD))</td><td align="left" valign="top" rowspan="1" colspan="1">43.1 (17.0)</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Gender, n (%)</td><td align="left" valign="top" rowspan="1" colspan="1"/></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Female</td><td align="left" valign="top" rowspan="1" colspan="1">1022 (56.6)</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Male</td><td align="left" valign="top" rowspan="1" colspan="1">777 (43.0)</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Other</td><td align="left" valign="top" rowspan="1" colspan="1">
<xref rid="TFN2" ref-type="table-fn">*</xref>
</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Race, n (%)</td><td align="left" valign="top" rowspan="1" colspan="1"/></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">White</td><td align="left" valign="top" rowspan="1" colspan="1">1443 (79.9)</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Black or African American</td><td align="left" valign="top" rowspan="1" colspan="1">129 (7.1)</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Other</td><td align="left" valign="top" rowspan="1" colspan="1">234 (13.0)</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Ethnicity, n (%)</td><td align="left" valign="top" rowspan="1" colspan="1"/></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Non-Hispanic</td><td align="left" valign="top" rowspan="1" colspan="1">1519 (84.1)</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Hispanic</td><td align="left" valign="top" rowspan="1" colspan="1">135 (7.5)</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Other</td><td align="left" valign="top" rowspan="1" colspan="1">152 (8.4)</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Primary language, n (%)</td><td align="left" valign="top" rowspan="1" colspan="1"/></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">English</td><td align="left" valign="top" rowspan="1" colspan="1">1696 (93.9)</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Spanish</td><td align="left" valign="top" rowspan="1" colspan="1">54 (3.0)</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Other</td><td align="left" valign="top" rowspan="1" colspan="1">56 (3.1)</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Marital status, n (%)</td><td align="left" valign="top" rowspan="1" colspan="1"/></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Non-union</td><td align="left" valign="top" rowspan="1" colspan="1">1080 (59.8)</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Union</td><td align="left" valign="top" rowspan="1" colspan="1">700 (38.8)</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Other</td><td align="left" valign="top" rowspan="1" colspan="1">246 (1.4)</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Seizure type at baseline, n (%)</td><td align="left" valign="top" rowspan="1" colspan="1"/></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Focal</td><td align="left" valign="top" rowspan="1" colspan="1">1059 (58.6)</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Generalized</td><td align="left" valign="top" rowspan="1" colspan="1">640 (35.5)</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Other</td><td align="left" valign="top" rowspan="1" colspan="1">107 (5.9)</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Insurance, n (%)</td><td align="left" valign="top" rowspan="1" colspan="1"/></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Private / commercial</td><td align="left" valign="top" rowspan="1" colspan="1">818 (45.3)</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Medicare / government</td><td align="left" valign="top" rowspan="1" colspan="1">636 (35.2)</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Other</td><td align="left" valign="top" rowspan="1" colspan="1">352 (19.5)</td></tr></tbody></table><table-wrap-foot><fn id="TFN1"><p id="P68">The number of patients is represented by n. SD &#x02013; standard deviation.</p></fn><fn id="TFN2"><label>*</label><p id="P69">A number less than 10 is omitted to preserve patient privacy</p></fn><fn id="TFN3"><label>(1)</label><p id="P70">Number of clinic visits with documented seizure control information including seizure type and frequency.</p></fn></table-wrap-foot></table-wrap><table-wrap position="float" id="T2"><label>Table 2</label><caption><p id="P71">Performance of the regex-based approach algorithm, compared to the gold standard of experts&#x02019; manual review for 100 clinical notes randomly selected.</p></caption><table frame="hsides" rules="groups"><colgroup span="1"><col align="left" valign="middle" span="1"/><col align="left" valign="middle" span="1"/><col align="left" valign="middle" span="1"/><col align="left" valign="middle" span="1"/><col align="left" valign="middle" span="1"/></colgroup><thead><tr><th align="left" valign="top" rowspan="1" colspan="1">Seizure control metric</th><th align="left" valign="top" rowspan="1" colspan="1">Number of items (algorithm/experts)</th><th align="left" valign="top" rowspan="1" colspan="1">Precision</th><th align="left" valign="top" rowspan="1" colspan="1">Recall</th><th align="left" valign="top" rowspan="1" colspan="1">F-score</th></tr></thead><tbody><tr><td align="left" valign="top" rowspan="1" colspan="1">Seizure type</td><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">(a, b, c, d)</td><td align="left" valign="top" rowspan="1" colspan="1">100/100, 52/52, 15/16, 2/2</td><td align="left" valign="top" rowspan="1" colspan="1">1, 1, 1, 1</td><td align="left" valign="top" rowspan="1" colspan="1">1, 1, 0.938, 1</td><td align="left" valign="top" rowspan="1" colspan="1">1, 1, 0.968, 1</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Diagnostic certainty</td><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">(a, b, c, d)</td><td align="left" valign="top" rowspan="1" colspan="1">100/100, 53/53, 18/18, 4/4</td><td align="left" valign="top" rowspan="1" colspan="1">1, 1, 1, 1</td><td align="left" valign="top" rowspan="1" colspan="1">1, 1, 1, 1</td><td align="left" valign="top" rowspan="1" colspan="1">1, 1, 1, 1</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Date of last seizure</td><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">(a, b, c, d)</td><td align="left" valign="top" rowspan="1" colspan="1">98/99, 50/50, 16/16, 2/2</td><td align="left" valign="top" rowspan="1" colspan="1">1, 1, 1, 1</td><td align="left" valign="top" rowspan="1" colspan="1">0.989, 1, 1, 1</td><td align="left" valign="top" rowspan="1" colspan="1">0.995, 1, 1, 1</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Seizure frequency</td><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">(a, b, c, d)</td><td align="left" valign="top" rowspan="1" colspan="1">100/100, 49/50, 16/16, 2/2</td><td align="left" valign="top" rowspan="1" colspan="1">1, 1, 1, 1</td><td align="left" valign="top" rowspan="1" colspan="1">1, 0.980, 1, 1</td><td align="left" valign="top" rowspan="1" colspan="1">1, 0.989, 1, 1</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Patient has epilepsy</td><td align="left" valign="top" rowspan="1" colspan="1">98/98</td><td align="left" valign="top" rowspan="1" colspan="1">1</td><td align="left" valign="top" rowspan="1" colspan="1">1</td><td align="left" valign="top" rowspan="1" colspan="1">1</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">ASM missed</td><td align="left" valign="top" rowspan="1" colspan="1">41/41</td><td align="left" valign="top" rowspan="1" colspan="1">1</td><td align="left" valign="top" rowspan="1" colspan="1">1</td><td align="left" valign="top" rowspan="1" colspan="1">1</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">ASM side effects</td><td align="left" valign="top" rowspan="1" colspan="1">41/41</td><td align="left" valign="top" rowspan="1" colspan="1">1</td><td align="left" valign="top" rowspan="1" colspan="1">1</td><td align="left" valign="top" rowspan="1" colspan="1">1</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">ASM adherence</td><td align="left" valign="top" rowspan="1" colspan="1">30/30</td><td align="left" valign="top" rowspan="1" colspan="1">1</td><td align="left" valign="top" rowspan="1" colspan="1">1</td><td align="left" valign="top" rowspan="1" colspan="1">1</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Total</td><td align="left" valign="top" rowspan="1" colspan="1">887/890</td><td align="left" valign="top" rowspan="1" colspan="1">1</td><td align="left" valign="top" rowspan="1" colspan="1">0.996</td><td align="left" valign="top" rowspan="1" colspan="1">0.998</td></tr></tbody></table><table-wrap-foot><fn id="TFN4"><p id="P72">ASM &#x02013; Anti-seizure medication.</p></fn></table-wrap-foot></table-wrap></floats-group></article>