<!DOCTYPE article
PUBLIC "-//NLM//DTD JATS (Z39.96) Journal Archiving and Interchange DTD with MathML3 v1.3 20210610//EN" "JATS-archivearticle1-3-mathml3.dtd">
<article xmlns:mml="http://www.w3.org/1998/Math/MathML" xmlns:xlink="http://www.w3.org/1999/xlink" dtd-version="1.3" article-type="research-article"><?properties manuscript?><processing-meta base-tagset="archiving" mathml-version="3.0" table-model="xhtml" tagset-family="jats"><restricted-by>pmc</restricted-by></processing-meta><front><journal-meta><journal-id journal-id-type="nlm-journal-id">8406899</journal-id><journal-id journal-id-type="pubmed-jr-id">7945</journal-id><journal-id journal-id-type="nlm-ta">Vaccine</journal-id><journal-id journal-id-type="iso-abbrev">Vaccine</journal-id><journal-title-group><journal-title>Vaccine</journal-title></journal-title-group><issn pub-type="ppub">0264-410X</issn><issn pub-type="epub">1873-2518</issn></journal-meta><article-meta><article-id pub-id-type="pmid">34716041</article-id><article-id pub-id-type="pmc">8653902</article-id><article-id pub-id-type="doi">10.1016/j.vaccine.2021.10.041</article-id><article-id pub-id-type="manuscript">NIHMS1754034</article-id><article-categories><subj-group subj-group-type="heading"><subject>Article</subject></subj-group></article-categories><title-group><article-title>Agreement among Sources of Adult Influenza Vaccination in the Age of
Immunization Information Systems</article-title></title-group><contrib-group><contrib contrib-type="author"><name><surname>Nowalk</surname><given-names>Mary Patricia</given-names></name><degrees>PhD, RDN</degrees><xref ref-type="aff" rid="A1">1</xref></contrib><contrib contrib-type="author"><name><surname>D&#x02019;Agostino</surname><given-names>Helen Aslanidou</given-names></name><degrees>MS</degrees><xref ref-type="aff" rid="A2">2</xref></contrib><contrib contrib-type="author"><name><surname>Zimmerman</surname><given-names>Richard K.</given-names></name><degrees>MD</degrees><xref ref-type="aff" rid="A3">3</xref></contrib><contrib contrib-type="author"><name><surname>Saul</surname><given-names>Sean G.</given-names></name><degrees>MPH</degrees><xref ref-type="aff" rid="A3">3</xref></contrib><contrib contrib-type="author"><name><surname>Susick</surname><given-names>Michael</given-names></name><degrees>MPH</degrees><xref ref-type="aff" rid="A3">3</xref></contrib><contrib contrib-type="author"><name><surname>Raviotta</surname><given-names>Jonathan M.</given-names></name><degrees>PhD</degrees><xref ref-type="aff" rid="A3">3</xref></contrib><contrib contrib-type="author"><name><surname>Sax</surname><given-names>Theresa M.</given-names></name><degrees>BS</degrees><xref ref-type="aff" rid="A2">2</xref></contrib><contrib contrib-type="author"><name><surname>Balasubramani</surname><given-names>G. K.</given-names></name><degrees>PhD</degrees><xref ref-type="aff" rid="A2">2</xref></contrib></contrib-group><aff id="A1"><label>1</label>University of Pittsburgh School of Medicine, Department of
Family Medicine, Suite 520 Schenley Place, 4420 Bayard St, Pittsburgh, PA 15260,
USA</aff><aff id="A2"><label>2</label>University of Pittsburgh Graduate School of Public Health,
Department of Epidemiology, Suite 600 Schenley Place, 4420 Bayard St, Pittsburgh, PA
15260, USA</aff><aff id="A3"><label>3</label>University of Pittsburgh School of Medicine, Department of
Family Medicine, Suite 520 Schenley Place, 4420 Bayard St, Pittsburgh, PA 15260,
USA</aff><author-notes><corresp id="CR1"><bold>Correspondence:</bold> Mary Patricia Nowalk, PhD, RDN, Suite
520 Schenley Place, 4420 Bayard Street, Pittsburgh, PA 15260, 412/383-2355,
<email>tnowalk@pitt.edu</email></corresp></author-notes><pub-date pub-type="nihms-submitted"><day>25</day><month>11</month><year>2021</year></pub-date><pub-date pub-type="epub"><day>27</day><month>10</month><year>2021</year></pub-date><pub-date pub-type="ppub"><day>16</day><month>11</month><year>2021</year></pub-date><pub-date pub-type="pmc-release"><day>16</day><month>11</month><year>2022</year></pub-date><volume>39</volume><issue>47</issue><fpage>6829</fpage><lpage>6836</lpage><!--elocation-id from pubmed: 10.1016/j.vaccine.2021.10.041--><abstract id="ABS1"><sec id="S1"><title>Introduction:</title><p id="P1">Many vaccination studies rely on self-reported vaccination status,
with its inherent biases. Accuracy of influenza vaccination self-report has
been evaluated periodically, typically using the medical record as the gold
standard. The burgeoning of electronic medical records (EMRs) and
immunization information systems (IISs) and the rise of adult vaccine
administration in community pharmacies suggest the need for a reevaluation
of self-reported vaccination status.</p></sec><sec id="S2"><title>Methods:</title><p id="P2">Vaccination data from self-report, the state IIS, the health system
EMR and other sources were compared for participants in outpatient and
inpatient influenza vaccine effectiveness studies for four seasons
(2016&#x02013;2017 to 2019&#x02013;2020). Agreement among the sources was
calculated along with sensitivity and specificity. Tests for trend assessed
changes in completeness of the Pennsylvania - Statewide IIS (PA-SIIS) data
over time.</p></sec><sec id="S3"><title>Results:</title><p id="P3">With self-report as the gold standard, agreement with the local EMR,
PA-SIIS, and all sources was 62%, 77% and 85%, respectively. Sensitivity of
the EMR was 42% (95% CI=41, 43) and specificity was 91% (90, 92). With
PA-SIIS-as the gold standard, agreement with the local EMR and all sources
was 77% and 78%, respectively. Sensitivity of all sources combined was 96%
(95, 97) and specificity was (63% (62, 64). Capture of influenza
vaccinations in the IIS has not consistently improved over time, with a
significant increase among children (<italic>P</italic>=0.001), no change
among working-age adults and a decrease among older adults
(<italic>P</italic>=0.004). However, PA-SIIS provided the largest
percentage of verified vaccines (69.3%) compared with EMR (43.3%) and other
sources (12.4%).</p></sec><sec id="S4"><title>Conclusion:</title><p id="P4">Both self-report and PA-SIIS are good estimates of actual vaccine
uptake. When high accuracy data are required, such as for vaccine
effectiveness studies, triangulation using multiple sources should be
conducted.</p></sec></abstract><kwd-group><kwd>Adult influenza vaccination</kwd><kwd>Immunization Information Systems</kwd><kwd>vaccination status</kwd><kwd>vaccine registries</kwd><kwd>electronic medical records</kwd></kwd-group></article-meta></front><body><sec id="S5"><title>Introduction</title><p id="P5">Self-report is frequently used to estimate vaccination coverage in national
and statewide surveys such as the National Immunization Survey (NIS) [<xref rid="R1" ref-type="bibr">1</xref>], National Health Interview Survey (NHIS) [<xref rid="R2" ref-type="bibr">2</xref>], and the Behavioral Risk Factor Surveillance
Survey (BRFSS) [<xref rid="R3" ref-type="bibr">3</xref>], as well as numerous
intervention and quality improvement studies. In the past, the medical record has
been used as the &#x0201c;gold standard&#x0201d; against which self-reported
vaccination has been measured. Studies by MacDonald et al. and Zimmerman et al.
reported sensitivity of 98%&#x02212;100% and specificity of 38%&#x02212;71% for
self-report of adult influenza vaccines [<xref rid="R4" ref-type="bibr">4</xref>,
<xref rid="R5" ref-type="bibr">5</xref>]. Newer studies using 2007&#x02013;2008
data, compared self-report and electronic medical records (EMRs), finding similar
agreement between them [<xref rid="R6" ref-type="bibr">6</xref>, <xref rid="R7" ref-type="bibr">7</xref>]. These studies were conducted before EMRs were mature
and widely used for example, in 2008, use of EMRs by healthcare providers was
estimated at 10% [<xref rid="R8" ref-type="bibr">8</xref>].</p><p id="P6">As part of the American Recovery and Reinvestment Act of 2009, expansion of
health technology including EMRs was encouraged and financed through the HITECH Act
[<xref rid="R9" ref-type="bibr">9</xref>] that provided incentives to health
care providers to adopt EMRs, in part to improve health care quality [<xref rid="R8" ref-type="bibr">8</xref>]. Since that time, considerable effort has been made to
convert paper charts to electronic systems, enable under resourced practices to
purchase EMR software, train vaccine providers how to consistently record vaccines
in the EMR, and use EMR software to prompt providers to vaccinate. Currently, 96% of
hospitals and 86% of office-based physicians use EMRs [<xref rid="R10" ref-type="bibr">10</xref>]. EMRs have distinct advantages over paper charts,
such as legibility, consistent recording in a designated location, and electronic
search capability that is more rapid and likely more accurate than manual chart
reviews.</p><p id="P7">Concomitantly, Immunization Information systems (IISs) or vaccine registries
have matured. IISs are defined as &#x0201c;confidential, population-based databases
of all immunization doses administered by participating providers to persons
residing within a given geopolitical area&#x0201d; [<xref rid="R11" ref-type="bibr">11</xref>]. These regional (usually statewide) repositories of vaccine
administrations have enormous value for ordering vaccines, increasing vaccinations
through patient level reminder and recall, provider assessment and feedback [<xref rid="R12" ref-type="bibr">12</xref>], and policy making. As of 2016, 55 U.S.
jurisdictions, including 49 states and 6 cities had implemented IISs; childhood
participation in IISs was 94% in 2016 [<xref rid="R13" ref-type="bibr">13</xref>]
and adult participation was 24.5% in 2012 [<xref rid="R14" ref-type="bibr">14</xref>]. Completeness of vaccine registries varies across jurisdictions because
they are governed by their individual jurisdictional policies and regulations [<xref rid="R11" ref-type="bibr">11</xref>]. Functional standards for IISs, which were
first developed in 2001 and revised several times since then, include having the
capability for bidirectional information exchange with EMRs to enhance accuracy of
both repositories [<xref rid="R15" ref-type="bibr">15</xref>]. However, the
Pennsylvania Statewide IIS (PA-SIIS) and 40 others do not mandate adult influenza
immunization reporting [<xref rid="R16" ref-type="bibr">16</xref>].</p><p id="P8">For over a decade, the Centers for Disease Control and Prevention (CDC) has
funded influenza vaccine effectiveness (VE) studies, namely, the outpatient U.S.
Influenza Vaccine Effectiveness Network (US Flu VE) and more recently, the inpatient
Hospitalized Influenza Vaccine Effectiveness Network (HAIVEN). These studies use a
test-negative case-control design to determine if the rate of influenza vaccination
differs among influenza cases compared with non-influenza acute respiratory cases.
Thus, validity of the VE estimates is dependent upon accurate assessment of
vaccination status. Investigators in both studies use a variety of sources for
obtaining vaccination status including self-report, electronic medical records
(EMR), and state immunization registries. VE studies are a newer use of state
immunization registries; their validity for this purpose has rarely been tested
[<xref rid="R12" ref-type="bibr">12</xref>, <xref rid="R17" ref-type="bibr">17</xref>].</p><p id="P9">The major changes in the immunization infrastructure environment, as well as
emerging uses for vaccination data warrant a reexamination of the accuracy of the
sources of immunization information. The purpose of this study is to determine
agreement among different sources of influenza vaccination data from participants in
the combined CDC outpatient and inpatient VE studies from a single local health
system over four influenza vaccination seasons, 2016&#x02013;2017 through
2019&#x02013;2020.</p></sec><sec id="S6"><title>Methods</title><p id="P10">The University of Pittsburgh Institutional Review Board approved this study.
Vaccination data were derived from an outpatient (US Flu VE Network) and an
inpatient (HAIVEN) test-negative, case-control influenza VE studies [<xref rid="R18" ref-type="bibr">18</xref>, <xref rid="R19" ref-type="bibr">19</xref>]. Consented
participants or parents of minor children completed an enrollment interview with
questions about receiving influenza vaccine during the season of enrollment that
included anytime since July 1. They were asked to provide information about where
and when they received the vaccine to assist with verification. A vaccine
information sheet and addressed, stamped envelope was given to those who stated that
they had information at home, but could not remember it during enrollment, to
complete and return to the study team. The vaccinations recorded at enrollment or
received in the mail were considered to be self-reported. At the end of enrollment
for each season, requests for influenza vaccination data for all participants were
made to the health system EMR and the Pennsylvania Statewide IIS (PA-SIIS). There
are established systems for requesting data from each of these sources, such as the
University of Pittsburgh&#x02019;s Clinical Translational Science Institute&#x02019;s
Health Record Research Request (R3) system that provides clinical data from the
health system&#x02019;s EMR for research studies.</p><p id="P11">Most clinics (doctor&#x02019;s offices, hospitals, health centers, mass
vaccination clinics, etc.) that report immunization data to the PA-SIIS use an HL7
interface (PA-SIIS personal communication 2021). This health system&#x02019;s EMR and
PA-SIIS use a bidirectional interface such that immunization data entered into the
EMR is &#x0201c;live-feed&#x0201d; uploaded to PA-SIIS; whereas, the health
system&#x02019;s EMR routinely queries PA-SIIS in batches to receive immunization
data reports that are then uploaded to the EMR. These reports are typically
automatically integrated into the immunization record</p><p id="P12">Finally, research assistants telephoned pharmacies, work sites, community
clinics, primary care providers, etc. to confirm self-reported vaccinations that
could not be confirmed in the EMR or PA-SIIS. Copies of the signed consent forms
were provided to these sources to facilitate release of the information. Verbal
vaccine verifications were recorded by research assistants or documentation was
faxed to the research office for recording. Vaccinations were considered verified if
they were located in PA-SIIS, the EMR or reported from one of the other sources.
Vaccinations that were recorded in the EMR or PA-SIIS but had not been reported by
the participant, were called &#x0201c;discovered&#x0201d; vaccinations. They are
reported herein, but were not included in the agreement analyses.</p><sec id="S7"><title>Statistical Analysis</title><p id="P13">Baseline measures by influenza season are reported using counts and
percentages. Differences in vaccination records across age and racial groups and
the differences in reporting between EMR and PA-SIIS (bi-directional data
transfer) were tested by Chi-square statistic. The latter comparisons were made
as a test of the quality of the bidirectional data transfer. We compared the
percent of self-reported influenza vaccines confirmed in PA-SIIS over the four
seasons to determine if reporting changed over time using the Cochran-Armitage
trend test [<xref rid="R20" ref-type="bibr">20</xref>] with two-sided
<bold>p</bold>-values.</p><p id="P14">We used two vaccination sources as the gold standard, self-report and
PA-SIIS in separate sets of analyses. The percent agreement between the gold
standard and the other measures was calculated as the number of vaccinated and
unvaccinated participnts that agreed in both sources divided by the total number
of participants. To assess the test&#x02019;s validity to discriminate between
two verification sources, we evaluated percent agreement, Cohen&#x02019;s kappa,
sensitivity, specificity, positive predictive value (PPV), negative predictive
value (NPV), and area under the ROC curve. These values were calculated for all
four seasons combined and by age group and race category. The kappa statistic
was interpreted as 0&#x02013;0.20 no agreement, 0.21&#x02013;0.39 minimal
agreement; 0.40-.059 weak agreement; 0.60&#x02013;0.79 moderate agreement;
0.80&#x02013;0.90 strong agreement and; &#x0003e;0.90 almost perfect agreement
[<xref rid="R21" ref-type="bibr">21</xref>].</p><p id="P15">We also compared the bidirectional transfer of immunization data between
PA-SIIS and the EMR by dividing the vaccinations recorded in both sources by: a)
the vaccines in PA-SIIS only; and b) the vaccines in the EMR. All statistical
analyses were carried out using SAS version 9.4 (SAS Inc., Cary, NC).</p></sec></sec><sec id="S8"><title>Results</title><p id="P16">A total of 9,930 participants were enrolled in the combined VE studies over
the four seasons. In general, approximately one fifth were children, over half (56%)
were adults 18&#x02013;64 years and about one fourth were older adults &#x02265;65
years old. Over half (57.5%) of the participants were female, 27.6% were non-white,
2.4% identified as Hispanic and 60% self-reported influenza vaccination. <xref rid="T1" ref-type="table">Table 1</xref> summarizes the demographic
characteristics of participants included in these analyses overall and for each
season.</p><p id="P17"><xref rid="F1" ref-type="fig">Figure 1</xref> shows the vaccination status
and source of vaccine information for the study population and <xref rid="T2" ref-type="table">Table 2</xref> shows the self-reported vaccinations, discovered
vaccinations, as well as the number and percent of vaccinations verified through the
various sources. Ten percent of vaccinations were &#x0201c;discovered,&#x0201d; that
is, vaccinations that were recorded among those who had reported not being
vaccinated. PA-SIIS represented the largest source of verified vaccinations (76.3%;
range = 72.9%&#x02013;92.5% across the various seasons and age groups), followed by
the EMR (47.7%; range = 40.6% &#x02013; 65.9%) and other sources (13.7%; range =
10.1% &#x02013; 16.4%). Combining all sources, 94.8% of self-reported vaccines could
be verified. The highest verification in both PA-SIIS and the EMR was for children
and the lowest verification in both sources was for working age adults
(<italic>P</italic>&#x02264;0.001 across all age groups). Non-white participants
compared with white participants had significantly higher verification in PA-SIIS
and the EMR (<italic>P</italic>&#x0003c;0.001).</p><p id="P18">Agreement analyses were conducted using: 1) self-report as the gold
standard; and 2) PA-SIIS as the gold standard, shown in <xref rid="T3" ref-type="table">Tables 3</xref> and <xref rid="T4" ref-type="table">4</xref>,
respectively. When self-report was compared with EMR records, agreement was low
(62%), as indicated by a kappa of 0.29 (minimal agreement). When self-report was
compared with PA-SIIS only, agreement increased to 77%, and kappa was 0.54 (weak
agreement). Using all possible sources of vaccination data agreement with
self-reported vaccinations rose to 85%, with a sensitivity and specificity of 85%
and kappa of 0.69 (moderate agreement). <xref rid="SD1" ref-type="supplementary-material">Supplemental Tables 1</xref>&#x02013;<xref rid="SD1" ref-type="supplementary-material">3</xref> show the agreement values
with self-report for age and race groups.</p><p id="P19">When PA-SIIS was used as the gold standard and compared with the EMR,
agreement was 77%, and kappa was 0.52. Adding self-report and other sources to the
EMR data resulted in agreement with PA-SIIS of 78%, sensitivity of 96%, specificity
of 63% and kappa of 0.57 (weak agreement). <xref rid="SD1" ref-type="supplementary-material">Supplemental Tables 4</xref> and <xref rid="SD1" ref-type="supplementary-material">5</xref> show the agreement values
with PA-SIIS for age and race groups.</p><p id="P20"><xref rid="F2" ref-type="fig">Figures 2a</xref> and <xref rid="F2" ref-type="fig">2b</xref> show the percent of self-reported influenza vaccines
confirmed in PA-SIIS over the four seasons. Capture of influenza vaccines in the
state registry increased significantly over time among children (test for trend
<italic>P</italic>=0.001), decreased significantly for older adults
(<italic>P</italic>=0.004), but did not change significantly among younger
adults, nor in either racial group. The transfer of data between the EMR and PA-SIIS
was further explored and is shown in <xref rid="T5" ref-type="table">Table 5</xref>.
The percent of PA-SIIS vaccines found in the EMR (transfer from EMR to PA-SIIS)
overall was high at nearly 90% and was 90% or higher for most seasons and age and
race groups. Notable exceptions were the 2018&#x02013;2019 season (77.7%),
working-age adults (85.4%) and non-white participants (86%). Over the four seasons,
transfer from the EMR to PA-SIIS increased significantly (<italic>P</italic>=0.036).
Conversely, the transfer of data from PA-SIIS to the EMR was considerably lower at
56.2% overall and the changes over time were not significant
(<italic>P</italic>=0.447).</p></sec><sec id="S9"><title>Discussion</title><p id="P21">This study compared several sources of influenza vaccination status that
have been used in influenza research studies. These sources include EMR, PA-SIIS,
other records and self-report. National and regional surveys of vaccine receipt to
determine vaccination coverage (NHIS, NIS, BRFSS) rely strictly on self-report, with
the general acknowledgement of the recall bias and response bias inherent to that
source of vaccination status. In influenza VE studies, self-reported vaccination
status has been verified using EMR data, IIS records, employee health records and
manual tracking by research assistants to confirm vaccination performed in private
physician&#x02019;s offices, pharmacies and health plan records, among others. This
effort to verify vaccination using all reasonable sources is made because the
accuracy of VE estimates is directly related to the accuracy of vaccination status.
Thus, understanding the relationships among these sources of influenza vaccination
information is essential.</p><p id="P22">Previous studies of influenza vaccine self-report among older adults
compared with medical charts and electronic databases as the &#x0201c;gold
standards&#x0201d; have reported sensitivity of 98%&#x02212;100% indicating that
self-report of receiving influenza vaccine was considered to be highly reliable;
whereas, self-report of not receiving influenza vaccine was not as reliable, with
specificity of 38%&#x02212;71% [<xref rid="R4" ref-type="bibr">4</xref>, <xref rid="R5" ref-type="bibr">5</xref>]. This level of agreement was confirmed in
more recent studies comparing the EMR and self-report [<xref rid="R6" ref-type="bibr">6</xref>, <xref rid="R7" ref-type="bibr">7</xref>]For example,
when the EMR was the gold standard, sensitivity of self-report was 93% and
specificity was 65% overall and among those age 65 years and older, sensitivity was
96% and specificity was 50% [<xref rid="R6" ref-type="bibr">6</xref>, <xref rid="R7" ref-type="bibr">7</xref>]. When self-report was the gold standard, sensitivity
of the EMR ranged from 51% to 89% [<xref rid="R6" ref-type="bibr">6</xref>, <xref rid="R7" ref-type="bibr">7</xref>];.In the current study, 10% of the total
influenza vaccinations had been reported by participants as not received, yet were
discovered in other sources including the EMR. In contrast, a study in the UK, where
nearly all vaccines are administered through the primary care office, 5% of
vaccinations reported in the EMR were self-reported as not received [<xref rid="R22" ref-type="bibr">22</xref>].</p><p id="P23">In this study, agreement between self-report and EMR alone was 62%, compared
with 68%, 89% and 92% agreement reported previously [<xref rid="R4" ref-type="bibr">4</xref>, <xref rid="R5" ref-type="bibr">5</xref>]. Agreement between
self-report and PA-SIIS was 77%, and agreement between self-report and combined EMR,
PA-SIIS and other sources was 85%.</p><p id="P24">These agreement values may reflect an increase in vaccination sites
available to this study&#x02019;s participants compared with previous studies. For
example, this study included a large proportion of working age adults who might
receive worksite influenza vaccines that we were unable to confirm or may reflect
the addition of pharmacy vaccination sites that has occurred since the licensure of
pharmacists as vaccinators. These sites may not routinely report vaccinations to the
IIS, thus reducing the number of vaccines that can be verified.</p><p id="P25">If self-report is not a suitable gold standard for influenza vaccination
status, is the IIS a viable option? Both EMRs and IISs have matured and become more
widely used [<xref rid="R13" ref-type="bibr">13</xref>] over the past two decades.
IISs are considered to be less prone to bias than other sources because they are
population based and are not limited to those who receive vaccines in medical
settings [<xref rid="R12" ref-type="bibr">12</xref>]. However, their level of
completeness may determine their usefulness for research. Previous studies have
examined the completeness of local IISs compared with EMRs and their suitability for
research. Five studies were identified, all of which included only children [<xref rid="R23" ref-type="bibr">23</xref>&#x02013;<xref rid="R27" ref-type="bibr">27</xref>]. Completeness ranged from 82% for influenza vaccination [<xref rid="R26" ref-type="bibr">26</xref>], to 97% [<xref rid="R25" ref-type="bibr">25</xref>] and 100% [<xref rid="R24" ref-type="bibr">24</xref>] for up-to-date
childhood vaccines in clinics that used direct electronic transfer of data from the
EMR to the IIS. Jackson et al. reported at least 97% match between the IIS and EMR
data for primary data elements including vaccine and date of administration [<xref rid="R27" ref-type="bibr">27</xref>]. The authors concluded that analyses using
IISs should be limited to only those variables that are IIS-required because they
have high internal validity, completeness and accuracy. A study conducted in Spain
in 2011&#x02013;2012 reported a 31% overestimation of influenza vaccination coverage
among adults by self-report compared with a computerized vaccination registry that
was linked to an EMR [<xref rid="R28" ref-type="bibr">28</xref>]. Additional
barriers to accurate record-keeping are identical names for different people or
multiple records for the same person with slightly or completely different
names.</p><p id="P26">Over four years, we did not find a consistent increase in the influenza
vaccination coverage reported in PA-SIIS. Influenza vaccination coverage increased
significantly in children, decreased significantly in older adults, and did not
change among working age adults, while coverage decreased in white participants and
increased in non-white participants. This lack of consistent improvement in coverage
may again reflect increased use of non-medical office vaccination sites that are not
required to report to PA-SIIS. Other sources of error may be imperfections in
bi-directionality of the transfer of vaccination data. We found that the transfer
from the EMR to PA-SIIS was high, but not so from PA-SIIS to the EMR. This disparity
may derive from the intricacies of creating interfaces that a) respect patient
privacy in the EMR; b) allow easy incorporation of IIS data into the EMR; and c)
direct the data to the appropriate place in the EMR where electronic data searches
of immunizations are made. Another, potential, factor that may impede data transfer
from the IIS to health system EMRs is the need to communicate with an assortment of
EMR software packages within its jurisdictional boundaries. Furthermore, as health
systems expand, they may have facilities in more than one IIS jurisdiction and
therefore have different reporting rules applied to them.</p><p id="P27">It is also possible that we did not observe increased adult vaccination
reporting to PA-SIIS over time because it has peaked and will not increase further
without changes to state laws that mandate adult vaccination reporting by all
vaccine administrators, and in other jurisdictions, remove opt in/opt out and
consent barriers to inclusion in the IIS.</p><p id="P28">Can the IIS be used as a reliable source of influenza immunization data for
research? A 2013 review of IIS use in public health research found only three VE
studies, none of which was for influenza [<xref rid="R12" ref-type="bibr">12</xref>]. A 2012 study of influenza VE was conducted among children using the New
York City Immunization Registry. The authors concluded that registry data can be
used to estimate VE in a pandemic when rapid turnaround in results is needed and the
IIS is well established [<xref rid="R17" ref-type="bibr">17</xref>]. When PA-SIIS
was used as the gold standard against the EMR and all other sources used for
influenza VE studies, agreement between the sources was modest indicating that IISs
may be acceptable sources of vaccination data for some studies, but when high
accuracy data are required, self-reported data, verified using multiple sources, may
be a better choice.</p><p id="P29">As of 2015, 96% (51) of the 53 IISs in the U.S. were authorized to collect
immunization data in all age groups, including adults, while only 12 had a mandate
to collect adult immunization data [<xref rid="R16" ref-type="bibr">16</xref>]. Some
IIS programs collecting adult immunization data allow adults to opt out, or require
explicit consent for vaccination data to be transferred to the IIS. These
stipulations potentially affect the quality and quantity of adult immunization data
in IISs across the country and may limit their usefulness as the gold standard for
vaccination information. Other efforts to improve IIS reporting might include IIS
reminder information included with every vaccine shipment, educational presentations
to businesses and agencies that offer vaccination services or host vaccination
clinics, and incentives to those groups for reporting to the IIS. Organizations such
as the Association of Immunization Registries of America (AIRA), the National Adult
and Influenza Immunization Summit (NAIIS) provide resources that promote
documentation of immunizations in IISs.</p><p id="P30">Since these data were collected, we have experienced a coronavirus pandemic
that resulted in the development of new COVID-19 vaccines. Unlike other adult
vaccines, reporting of all COVID-19 vaccines is required. This requirement should
result in significantly better capture of these vaccinations by the IISs.
Furthermore, as of this writing, countless businesses, agencies and events require
proof of vaccination before admission, thus enhancing self-report accuracy. Future
research comparing PA-SIIS as the gold standard for COVID-19 vaccines may
demonstrate the value of mandatory vaccine reporting to state immunization
information systems.</p><sec id="S10"><title>Strengths and Limitations</title><p id="P31">Data from two influenza VE studies that used nearly identical means to
assess self-reported vaccination status over four influenza seasons were
combined to increase sample size. There may have been other sources of data that
were not used to confirm self-report because the return on the time investment
was too low. Research indicates that over 40% of adults receive the influenza
vaccine in non-medical settings [<xref rid="R29" ref-type="bibr">29</xref>]. If
reporting of adult and/or influenza vaccines is not mandatory for these
settings, many self-reported vaccinations may not be verified in the IIS. The
findings in this study may not be generalizable to all locales. For example,
some states may have stricter reporting policies which may result in higher
agreement between self-report and the IIS, or may not include adults at all in
the IIS. By comparing PA-SIIS influenza vaccine coverage over four years, we
found that in Pennsylvania, childhood reporting continues to improve, but not
adult reporting. The interfaces between EMRs and IISs are continually improving
and past results may not correspond with current functionality.</p></sec><sec id="S11"><title>Conclusions</title><p id="P32">Both self-report and IIS provide reasonable estimates of actual vaccine
uptake. While IISs may provide valid and highly accurate reports of childhood
influenza vaccinations, their use for adult influenza research may be less
reliable without reporting mandates. When high accuracy data are required, such
as for vaccine effectiveness studies, triangulation using multiple sources
should be conducted.</p></sec></sec><sec sec-type="supplementary-material" id="SM1"><title>Supplementary Material</title><supplementary-material content-type="local-data" id="SD1"><label>1</label><media xlink:href="NIHMS1754034-supplement-1.docx" orientation="portrait" id="d40e432" position="anchor"/></supplementary-material></sec></body><back><ack id="S12"><title>Funding:</title><p id="P33">This work was supported by the Centers for Disease Control and Prevention
(CDC) [5U01IP001035] and by National Institutes of Health (NIH) [UL1TR001857]. This
work represents the views of the authors and not the CDC or NIH. It is subject to
CDC&#x02019;s public access policy.</p><sec id="S14"><title>Competing interests:</title><p id="P35">RKZ and JMR have research funding from Sanofi Pasteur. MPN, GKB, TMS and
HAD have research funding from Merck &#x00026; Co., Inc. on an unrelated project.
MPN has participated in the Merck pneumococcal expert forum. SGS and MS have no
conflicts to report. GKB serves as an advisor or consultant at New World Medical
Inc., Ocular Therapeutix, Inc., and Sight Sciences, Inc.</p></sec></ack><fn-group><fn id="FN1"><p id="P36" content-type="publisher-disclaimer">This is a PDF file of an unedited
manuscript that has been accepted for publication. As a service to our customers
we are providing this early version of the manuscript. The manuscript will
undergo copyediting, typesetting, and review of the resulting proof before it is
published in its final form. Please note that during the production process
errors may be discovered which could affect the content, and all legal
disclaimers that apply to the journal pertain.</p></fn></fn-group><ref-list><title>Literature Cited</title><ref id="R1"><label>1.</label><mixed-citation publication-type="web"><source>Increased Interseasonal Respiratory
Syncytial Virus (RSV) Activity in Parts of the Southern United
States</source> [<comment><ext-link ext-link-type="uri" xlink:href="https://emergency.cdc.gov/han/2021/han00443.asp">https://emergency.cdc.gov/han/2021/han00443.asp</ext-link></comment>]</mixed-citation></ref><ref id="R2"><label>2.</label><mixed-citation publication-type="web"><source>About the National Health Interview
Survey</source> [<comment><ext-link ext-link-type="uri" xlink:href="https://www.cdc.gov/nchs/nhis/about_nhis.htm">https://www.cdc.gov/nchs/nhis/about_nhis.htm</ext-link></comment>]</mixed-citation></ref><ref id="R3"><label>3.</label><mixed-citation publication-type="web"><source>Behavioral Risk Factor Surveillance
System (BRFSS)</source> [<comment><ext-link ext-link-type="uri" xlink:href="https://www.cdc.gov/brfss/">https://www.cdc.gov/brfss/</ext-link></comment>]</mixed-citation></ref><ref id="R4"><label>4.</label><mixed-citation publication-type="journal"><name><surname>Mac</surname><given-names>Donald R</given-names></name>, <name><surname>Baken</surname><given-names>L</given-names></name>, <name><surname>Nelson</surname><given-names>A</given-names></name>, <name><surname>Nichol</surname><given-names>KL</given-names></name>: <article-title>Validation of self-report of influenza and pneumococcal
vaccination status in elderly outpatients</article-title>. <source>Am J Prev
Med</source>
<year>1999</year>,
<volume>16</volume>(<issue>3</issue>):<fpage>173</fpage>&#x02013;<lpage>177</lpage>.<pub-id pub-id-type="pmid">10198654</pub-id></mixed-citation></ref><ref id="R5"><label>5.</label><mixed-citation publication-type="journal"><name><surname>Zimmerman</surname><given-names>RK</given-names></name>, <name><surname>Raymund</surname><given-names>M</given-names></name>, <name><surname>Janosky</surname><given-names>JE</given-names></name>, <name><surname>Nowalk</surname><given-names>MP</given-names></name>, <name><surname>Fine</surname><given-names>MJ</given-names></name>: <article-title>Sensitivity and specificity of patient self-report of
influenza and pneumococcal polysaccharide vaccinations among elderly
outpatients in diverse patient care strata</article-title>.
<source>Vaccine</source>
<year>2003</year>,
<volume>21</volume>(<issue>13&#x02013;14</issue>):<fpage>1486</fpage>&#x02013;<lpage>1491</lpage>.<pub-id pub-id-type="pmid">12615445</pub-id></mixed-citation></ref><ref id="R6"><label>6.</label><mixed-citation publication-type="journal"><name><surname>Greene</surname><given-names>SK</given-names></name>, <name><surname>Shi</surname><given-names>P</given-names></name>, <name><surname>Dutta-Linn</surname><given-names>MM</given-names></name>, <name><surname>Shoup</surname><given-names>JA</given-names></name>, <name><surname>Hinrichsen</surname><given-names>VL</given-names></name>, <name><surname>Ray</surname><given-names>P</given-names></name>, <name><surname>Nordin</surname><given-names>JD</given-names></name>, <name><surname>Kuckler</surname><given-names>L</given-names></name>, <name><surname>Weintraub</surname><given-names>ES</given-names></name>, <name><surname>Yih</surname><given-names>WK</given-names></name>: <article-title>Accuracy of data on influenza vaccination status at four
Vaccine Safety Datalink sites</article-title>. <source>Am J Prev
Med</source>
<year>2009</year>,
<volume>37</volume>(<issue>6</issue>):<fpage>552</fpage>&#x02013;<lpage>555</lpage>.<pub-id pub-id-type="pmid">19944924</pub-id></mixed-citation></ref><ref id="R7"><label>7.</label><mixed-citation publication-type="journal"><name><surname>Rolnick</surname><given-names>SJ</given-names></name>, <name><surname>Parker</surname><given-names>ED</given-names></name>, <name><surname>Nordin</surname><given-names>JD</given-names></name>, <name><surname>Hedblom</surname><given-names>BD</given-names></name>, <name><surname>Wei</surname><given-names>F</given-names></name>, <name><surname>Kerby</surname><given-names>T</given-names></name>, <name><surname>Jackson</surname><given-names>JM</given-names></name>, <name><surname>Crain</surname><given-names>AL</given-names></name>, <name><surname>Euler</surname><given-names>G</given-names></name>: <article-title>Self-report compared to electronic medical record across
eight adult vaccines: do results vary by demographic
factors?</article-title>
<source>Vaccine</source>
<year>2013</year>,
<volume>31</volume>(<issue>37</issue>):<fpage>3928</fpage>&#x02013;<lpage>3935</lpage>.<pub-id pub-id-type="pmid">23806243</pub-id></mixed-citation></ref><ref id="R8"><label>8.</label><mixed-citation publication-type="web"><source>Congressional Intent for the HITECH
Act</source> [<comment><ext-link ext-link-type="uri" xlink:href="https://www.ajmc.com/view/ajmc_10dechit_stark_sp24tp28?p=1">https://www.ajmc.com/view/ajmc_10dechit_stark_sp24tp28?p=1</ext-link></comment>]</mixed-citation></ref><ref id="R9"><label>9.</label><mixed-citation publication-type="web"><source>Health Information Technology for
Economic and Clinical Health (HITECH) Act. In</source>.;
<year>2009</year>.</mixed-citation></ref><ref id="R10"><label>10.</label><mixed-citation publication-type="web"><source>Health IT Dashboard</source>
[<comment><ext-link ext-link-type="uri" xlink:href="https://dashboard.healthit.gov/quickstats/quickstats.php">https://dashboard.healthit.gov/quickstats/quickstats.php</ext-link></comment>]</mixed-citation></ref><ref id="R11"><label>11.</label><mixed-citation publication-type="web"><source>Immunization Information Systems
(IIS)</source> [<comment><ext-link ext-link-type="uri" xlink:href="https://www.cdc.gov/vaccines/programs/iis/about.html">https://www.cdc.gov/vaccines/programs/iis/about.html</ext-link></comment>]</mixed-citation></ref><ref id="R12"><label>12.</label><mixed-citation publication-type="journal"><name><surname>Curran</surname><given-names>EA</given-names></name>, <name><surname>Bednarczyk</surname><given-names>RA</given-names></name>, <name><surname>Omer</surname><given-names>SB</given-names></name>: <article-title>Evaluation of the frequency of immunization information
system use for public health research</article-title>. <source>Hum Vaccin
Immunother</source>
<year>2013</year>,
<volume>9</volume>(<issue>6</issue>):<fpage>1346</fpage>&#x02013;<lpage>1350</lpage>.<pub-id pub-id-type="pmid">23422024</pub-id></mixed-citation></ref><ref id="R13"><label>13.</label><mixed-citation publication-type="journal"><name><surname>Murthy</surname><given-names>N</given-names></name>, <name><surname>Rodgers</surname><given-names>L</given-names></name>, <name><surname>Pabst</surname><given-names>L</given-names></name>, <name><surname>Fiebelkorn</surname><given-names>AP</given-names></name>, <name><surname>Ng</surname><given-names>T</given-names></name>: <article-title>Progress in Childhood Vaccination Data in Immunization
Information Systems - United States, 2013&#x02013;2016</article-title>.
<source>MMWR Morb Mortal Wkly Rep</source>
<year>2017</year>,
<volume>66</volume>(<issue>43</issue>):<fpage>1178</fpage>&#x02013;<lpage>1181</lpage>.<pub-id pub-id-type="pmid">29095809</pub-id></mixed-citation></ref><ref id="R14"><label>14.</label><mixed-citation publication-type="journal"><collab>Centers for Disease Control and
Prevention (CDC)</collab>: <article-title>Progress in immunization
information systems - United States, 2012</article-title>. <source>MMWR Morb
Mortal Wkly Rep</source>
<year>2013</year>,
<volume>62</volume>(<issue>49</issue>):<fpage>1005</fpage>&#x02013;<lpage>1008</lpage>.<pub-id pub-id-type="pmid">24336133</pub-id></mixed-citation></ref><ref id="R15"><label>15.</label><mixed-citation publication-type="web"><source>Immunization Information Systems
(IIS) Functional Standards</source> [<comment><ext-link ext-link-type="uri" xlink:href="https://www.cdc.gov/vaccines/programs/iis/functional-standards/func-stds-v4-0.html">https://www.cdc.gov/vaccines/programs/iis/functional-standards/func-stds-v4-0.html</ext-link></comment>]</mixed-citation></ref><ref id="R16"><label>16.</label><mixed-citation publication-type="journal"><name><surname>Martin</surname><given-names>DW</given-names></name>, <name><surname>Lowery</surname><given-names>NE</given-names></name>, <name><surname>Brand</surname><given-names>B</given-names></name>, <name><surname>Gold</surname><given-names>R</given-names></name>, <name><surname>Horlick</surname><given-names>G</given-names></name>: <article-title>Immunization information systems: a decade of progress
in law and policy</article-title>. <source>J Public Health Manag
Pract</source>
<year>2015</year>,
<volume>21</volume>(<issue>3</issue>):<fpage>296</fpage>&#x02013;<lpage>303</lpage>.<pub-id pub-id-type="pmid">24402434</pub-id></mixed-citation></ref><ref id="R17"><label>17.</label><mixed-citation publication-type="journal"><name><surname>Hadler</surname><given-names>JL</given-names></name>, <name><surname>Baker</surname><given-names>TN</given-names></name>, <name><surname>Papadouka</surname><given-names>V</given-names></name>, <name><surname>France</surname><given-names>AM</given-names></name>, <name><surname>Zimmerman</surname><given-names>C</given-names></name>, <name><surname>Livingston</surname><given-names>KA</given-names></name>, <name><surname>Zucker</surname><given-names>JR</given-names></name>: <article-title>Effectiveness of 1 dose of 2009 influenza A (H1N1)
vaccine at preventing hospitalization with pandemic H1N1 influenza in
children aged 7 months-9 years</article-title>. <source>J Infect
Dis</source>
<year>2012</year>,
<volume>206</volume>(<issue>1</issue>):<fpage>49</fpage>&#x02013;<lpage>55</lpage>.<pub-id pub-id-type="pmid">22551808</pub-id></mixed-citation></ref><ref id="R18"><label>18.</label><mixed-citation publication-type="journal"><name><surname>Ohmit</surname><given-names>SE</given-names></name>, <name><surname>Thompson</surname><given-names>MG</given-names></name>, <name><surname>Petrie</surname><given-names>JG</given-names></name>, <name><surname>Thaker</surname><given-names>SN</given-names></name>, <name><surname>Jackson</surname><given-names>ML</given-names></name>, <name><surname>Belongia</surname><given-names>EA</given-names></name>, <name><surname>Zimmerman</surname><given-names>RK</given-names></name>, <name><surname>Gaglani</surname><given-names>M</given-names></name>, <name><surname>Lamerato</surname><given-names>L</given-names></name>, <name><surname>Spencer</surname><given-names>SM</given-names></name>
<etal/>: <article-title>Influenza vaccine effectiveness in the 2011&#x02013;2012
season: protection against each circulating virus and the effect of prior
vaccination on estimates</article-title>. <source>Clin Infect Dis</source>
<year>2014</year>,
<volume>58</volume>(<issue>3</issue>):<fpage>319</fpage>&#x02013;<lpage>327</lpage>.<pub-id pub-id-type="pmid">24235265</pub-id></mixed-citation></ref><ref id="R19"><label>19.</label><mixed-citation publication-type="journal"><name><surname>Ferdinands</surname><given-names>JM</given-names></name>, <name><surname>Gaglani</surname><given-names>M</given-names></name>, <name><surname>Martin</surname><given-names>ET</given-names></name>, <name><surname>Middleton</surname><given-names>D</given-names></name>, <name><surname>Monto</surname><given-names>AS</given-names></name>, <name><surname>Murthy</surname><given-names>K</given-names></name>, <name><surname>Silveira</surname><given-names>FP</given-names></name>, <name><surname>Talbot</surname><given-names>HK</given-names></name>, <name><surname>Zimmerman</surname><given-names>R</given-names></name>, <name><surname>Alyanak</surname><given-names>E</given-names></name>
<etal/>: <article-title>Prevention of Influenza Hospitalization Among Adults in
the United States, 2015&#x02013;2016: Results From the US Hospitalized Adult
Influenza Vaccine Effectiveness Network (HAIVEN)</article-title>. <source>J
Infect Dis</source>
<year>2019</year>,
<volume>220</volume>(<issue>8</issue>):<fpage>1265</fpage>&#x02013;<lpage>1275</lpage>.<pub-id pub-id-type="pmid">30561689</pub-id></mixed-citation></ref><ref id="R20"><label>20.</label><mixed-citation publication-type="journal"><name><surname>Agresti</surname><given-names>A</given-names></name>: <article-title>A Survey of Exact Inference for Contingency
Tables</article-title>. <source>Statistical Science</source>
<year>1992</year>,
<volume>7</volume>(<issue>1</issue>):<fpage>131</fpage>&#x02013;<lpage>153</lpage>.</mixed-citation></ref><ref id="R21"><label>21.</label><mixed-citation publication-type="journal"><name><surname>McHugh</surname><given-names>ML</given-names></name>: <article-title>Interrater reliability: the kappa
statistic</article-title>. <source>Biochem Med (Zagreb)</source>
<year>2012</year>,
<volume>22</volume>(<issue>3</issue>):<fpage>276</fpage>&#x02013;<lpage>282</lpage>.<pub-id pub-id-type="pmid">23092060</pub-id></mixed-citation></ref><ref id="R22"><label>22.</label><mixed-citation publication-type="journal"><name><surname>Mangtani</surname><given-names>P</given-names></name>, <name><surname>Shah</surname><given-names>A</given-names></name>, <name><surname>Roberts</surname><given-names>JA</given-names></name>: <article-title>Validation of influenza and pneumococcal vaccine status
in adults based on self-report</article-title>. <source>Epidemiol
Infect</source>
<year>2007</year>,
<volume>135</volume>(<issue>1</issue>):<fpage>139</fpage>&#x02013;<lpage>143</lpage>.<pub-id pub-id-type="pmid">16740194</pub-id></mixed-citation></ref><ref id="R23"><label>23.</label><mixed-citation publication-type="journal"><name><surname>Boyd</surname><given-names>TD</given-names></name>, <name><surname>Linkins</surname><given-names>RW</given-names></name>, <name><surname>Mason</surname><given-names>K</given-names></name>, <name><surname>Bulim</surname><given-names>I</given-names></name>, <name><surname>Lemke</surname><given-names>B</given-names></name>: <article-title>Assessing immunization registry data completeness in
Bexar County, Texas</article-title>. <source>Am J Prev Med</source>
<year>2002</year>,
<volume>22</volume>(<issue>3</issue>):<fpage>184</fpage>&#x02013;<lpage>187</lpage>.<pub-id pub-id-type="pmid">11897463</pub-id></mixed-citation></ref><ref id="R24"><label>24.</label><mixed-citation publication-type="journal"><name><surname>Kolasa</surname><given-names>MS</given-names></name>, <name><surname>Chilkatowsky</surname><given-names>AP</given-names></name>, <name><surname>Clarke</surname><given-names>KR</given-names></name>, <name><surname>Lutz</surname><given-names>JP</given-names></name>: <article-title>How complete are immunization registries? The
Philadelphia story</article-title>. <source>Ambul Pediatr</source>
<year>2006</year>,
<volume>6</volume>(<issue>1</issue>):<fpage>21</fpage>&#x02013;<lpage>24</lpage>.<pub-id pub-id-type="pmid">16443179</pub-id></mixed-citation></ref><ref id="R25"><label>25.</label><mixed-citation publication-type="journal"><name><surname>Koepke</surname><given-names>R</given-names></name>, <name><surname>Petit</surname><given-names>AB</given-names></name>, <name><surname>Ayele</surname><given-names>RA</given-names></name>, <name><surname>Eickhoff</surname><given-names>JC</given-names></name>, <name><surname>Schauer</surname><given-names>SL</given-names></name>, <name><surname>Verdon</surname><given-names>MJ</given-names></name>, <name><surname>Hopfensperger</surname><given-names>DJ</given-names></name>, <name><surname>Conway</surname><given-names>JH</given-names></name>, <name><surname>Davis</surname><given-names>JP</given-names></name>: <article-title>Completeness and accuracy of the wisconsin immunization
registry: an evaluation coinciding with the beginning of meaningful
use</article-title>. <source>J Public Health Manag Pract</source>
<year>2015</year>,
<volume>21</volume>(<issue>3</issue>):<fpage>273</fpage>&#x02013;<lpage>281</lpage>.<pub-id pub-id-type="pmid">25590511</pub-id></mixed-citation></ref><ref id="R26"><label>26.</label><mixed-citation publication-type="journal"><name><surname>Hendrickson</surname><given-names>BK</given-names></name>, <name><surname>Panchanathan</surname><given-names>SS</given-names></name>, <name><surname>Petitti</surname><given-names>D</given-names></name>: <article-title>Evaluation of immunization data completeness within a
large community health care system exchanging data with a state immunization
information system</article-title>. <source>J Public Health Manag
Pract</source>
<year>2015</year>,
<volume>21</volume>(<issue>3</issue>):<fpage>288</fpage>&#x02013;<lpage>295</lpage>.<pub-id pub-id-type="pmid">24378608</pub-id></mixed-citation></ref><ref id="R27"><label>27.</label><mixed-citation publication-type="journal"><name><surname>Jackson</surname><given-names>ML</given-names></name>, <name><surname>Henrikson</surname><given-names>NB</given-names></name>, <name><surname>Grossman</surname><given-names>DC</given-names></name>: <article-title>Evaluating Washington State&#x02019;s immunization
information system as a research tool.</article-title>
<source>Acad Pediatr</source>
<year>2014</year>,
<volume>14</volume>(<issue>1</issue>):<fpage>71</fpage>&#x02013;<lpage>76</lpage>.<pub-id pub-id-type="pmid">24369871</pub-id></mixed-citation></ref><ref id="R28"><label>28.</label><mixed-citation publication-type="journal"><name><surname>Jimenez-Garcia</surname><given-names>R</given-names></name>, <name><surname>Hernandez-Barrera</surname><given-names>V</given-names></name>, <name><surname>Rodriguez-Rieiro</surname><given-names>C</given-names></name>, <name><surname>Carrasco</surname><given-names>Garrido P</given-names></name>, <name><surname>Lopez de Andres</surname><given-names>A</given-names></name>, <name><surname>Jimenez-Trujillo</surname><given-names>I</given-names></name>, <name><surname>Esteban-Vasallo</surname><given-names>MD</given-names></name>, <name><surname>Dominguez-Berjon</surname><given-names>MF</given-names></name>, <name><surname>de Miguel-Diez</surname><given-names>J</given-names></name>, <name><surname>Astray-Mochales</surname><given-names>J</given-names></name>: <article-title>Comparison of self-report influenza vaccination coverage
with data from a population based computerized vaccination registry and
factors associated with discordance</article-title>.
<source>Vaccine</source>
<year>2014</year>,
<volume>32</volume>(<issue>35</issue>):<fpage>4386</fpage>&#x02013;<lpage>4392</lpage>.<pub-id pub-id-type="pmid">24968159</pub-id></mixed-citation></ref><ref id="R29"><label>29.</label><mixed-citation publication-type="web"><source>National and State-Level Place of Flu
Vaccination among Vaccinated Adults in the United States, 2014&#x02013;15 Flu
Season</source> [<comment><ext-link ext-link-type="uri" xlink:href="https://www.cdc.gov/flu/fluvaxview/place-vaccination-2014-15.htm">https://www.cdc.gov/flu/fluvaxview/place-vaccination-2014-15.htm</ext-link></comment>]</mixed-citation></ref></ref-list></back><floats-group><fig id="F1" orientation="portrait" position="float"><label>Figure 1.</label><caption><p id="P37">Sources of vaccination data in study population</p></caption><graphic xlink:href="nihms-1754034-f0001"/></fig><fig id="F2" orientation="portrait" position="float"><label>Figure 2a and 2b.</label><caption><p id="P38">Percent of self-reported influenza vaccinations captured in PA-SIIS each
season by age group (2a) and race (2b)</p></caption><graphic xlink:href="nihms-1754034-f0002"/><graphic xlink:href="nihms-1754034-f0003"/></fig><table-wrap id="T1" position="float" orientation="landscape"><label>Table 1.</label><caption><p id="P39">Baseline characteristics of participants by influenza season
2016&#x02013;2017 through 2019&#x02013;2020</p></caption><table frame="above" rules="none"><colgroup span="1"><col align="left" valign="middle" span="1"/><col align="left" valign="middle" span="1"/><col align="left" valign="middle" span="1"/><col align="left" valign="middle" span="1"/><col align="left" valign="middle" span="1"/><col align="left" valign="middle" span="1"/></colgroup><thead><tr><th align="left" valign="top" rowspan="1" colspan="1"/><th align="center" valign="top" colspan="4" rowspan="1">Season</th><th align="left" valign="top" rowspan="1" colspan="1"/></tr><tr><th align="left" valign="bottom" rowspan="1" colspan="1">Characteristics</th><th align="center" valign="bottom" rowspan="1" colspan="1">2016&#x02013;2017 (N=2294)</th><th align="center" valign="bottom" rowspan="1" colspan="1">2017&#x02013;2018 (N=2324)</th><th align="center" valign="bottom" rowspan="1" colspan="1">2018&#x02013;2019 (N=2839)</th><th align="center" valign="bottom" rowspan="1" colspan="1">2019&#x02013;2020 (N=2473)</th><th align="center" valign="bottom" rowspan="1" colspan="1">Overall (N=9930)</th></tr><tr><th align="left" valign="top" rowspan="1" colspan="1"/><th align="center" valign="top" rowspan="1" colspan="1">n (%)</th><th align="center" valign="top" rowspan="1" colspan="1">n (%)</th><th align="center" valign="top" rowspan="1" colspan="1">n (%)</th><th align="center" valign="top" rowspan="1" colspan="1">n (%)</th><th align="center" valign="top" rowspan="1" colspan="1">n (%)</th></tr><tr><th align="left" valign="top" colspan="6" rowspan="1">
<hr/>
</th></tr></thead><tbody><tr><td align="left" valign="top" rowspan="1" colspan="1">Age Group</td><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">&#x02003;6 months - 17 years</td><td align="center" valign="top" rowspan="1" colspan="1">417 (18.2)</td><td align="center" valign="top" rowspan="1" colspan="1">399 (17.2)</td><td align="center" valign="top" rowspan="1" colspan="1">686 (24.2)</td><td align="center" valign="top" rowspan="1" colspan="1">534 (21.7)</td><td align="center" valign="top" rowspan="1" colspan="1">2036 (20.5)</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">&#x02003;18 &#x02013; 64 years</td><td align="center" valign="top" rowspan="1" colspan="1">1250 (54.5)</td><td align="center" valign="top" rowspan="1" colspan="1">1360 (58.6)</td><td align="center" valign="top" rowspan="1" colspan="1">1552 (54.7)</td><td align="center" valign="top" rowspan="1" colspan="1">1387 (56.3)</td><td align="center" valign="top" rowspan="1" colspan="1">5549 (56.0)</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">&#x02003;&#x02265;65 years</td><td align="center" valign="top" rowspan="1" colspan="1">627 (27.3)</td><td align="center" valign="top" rowspan="1" colspan="1">563 (24.2)</td><td align="center" valign="top" rowspan="1" colspan="1">601 (21.2)</td><td align="center" valign="top" rowspan="1" colspan="1">541 (22.0)</td><td align="center" valign="top" rowspan="1" colspan="1">2332 (23.5)</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Female sex, ref. = male</td><td align="center" valign="top" rowspan="1" colspan="1">1300 (56.7)</td><td align="center" valign="top" rowspan="1" colspan="1">1316 (56.7)</td><td align="center" valign="top" rowspan="1" colspan="1">1644 (57.9)</td><td align="center" valign="top" rowspan="1" colspan="1">1439 (58.5)</td><td align="center" valign="top" rowspan="1" colspan="1">5699 (57.5)</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Non-white race, ref. = white</td><td align="center" valign="top" rowspan="1" colspan="1">519 (22.7)</td><td align="center" valign="top" rowspan="1" colspan="1">635 (27.6)</td><td align="center" valign="top" rowspan="1" colspan="1">912 (32.4)</td><td align="center" valign="top" rowspan="1" colspan="1">652 (26.4)</td><td align="center" valign="top" rowspan="1" colspan="1">2718 (27.6)</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Hispanic ethnicity, ref. = non-Hispanic</td><td align="center" valign="top" rowspan="1" colspan="1">45 (2.0)</td><td align="center" valign="top" rowspan="1" colspan="1">48 (2.1)</td><td align="center" valign="top" rowspan="1" colspan="1">81 (2.9)</td><td align="center" valign="top" rowspan="1" colspan="1">62 (2.5)</td><td align="center" valign="top" rowspan="1" colspan="1">236 (2.4)</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Self-reported vaccination</td><td align="center" valign="top" rowspan="1" colspan="1">1427 (62.2)</td><td align="center" valign="top" rowspan="1" colspan="1">1362 (58.6)</td><td align="center" valign="top" rowspan="1" colspan="1">1679 (59.1)</td><td align="center" valign="top" rowspan="1" colspan="1">1476 (59.7)</td><td align="center" valign="top" rowspan="1" colspan="1">5944 (59.9)</td></tr></tbody></table></table-wrap><table-wrap id="T2" position="float" orientation="landscape"><label>Table 2.</label><caption><p id="P40">Recorded influenza vaccinations from various sources for all
participants for influenza seasons 2016&#x02013;2017 through 2019&#x02013;2020</p></caption><table frame="box" rules="none"><colgroup span="1"><col align="left" valign="middle" span="1"/><col align="left" valign="middle" span="1"/><col align="left" valign="middle" span="1"/><col align="left" valign="middle" span="1"/><col align="left" valign="middle" span="1"/><col align="left" valign="middle" span="1"/><col align="left" valign="middle" span="1"/></colgroup><thead><tr><th align="left" valign="top" rowspan="1" colspan="1"/><th align="center" valign="top" colspan="6" rowspan="1">Vaccinations verified
through</th></tr><tr><th align="center" valign="bottom" rowspan="1" colspan="1">Season/group</th><th align="center" valign="bottom" rowspan="1" colspan="1">Self-reported vaccination N</th><th align="center" valign="bottom" rowspan="1" colspan="1">PA-SIIS<sup><xref rid="TFN1" ref-type="table-fn">a</xref></sup> N (%)</th><th align="center" valign="bottom" rowspan="1" colspan="1">EMR<sup><xref rid="TFN2" ref-type="table-fn">b</xref></sup> N (%)</th><th align="center" valign="bottom" rowspan="1" colspan="1">Other sources<sup><xref rid="TFN3" ref-type="table-fn">c</xref></sup> N (%)</th><th align="center" valign="bottom" rowspan="1" colspan="1">All sources<sup><xref rid="TFN4" ref-type="table-fn">d</xref></sup> N (%)</th><th align="center" valign="bottom" rowspan="1" colspan="1">Discovered vaccination<sup><xref rid="TFN5" ref-type="table-fn">e</xref></sup> N (%)</th></tr><tr><th align="left" valign="top" colspan="7" rowspan="1">
<hr/>
</th></tr></thead><tbody><tr><td align="left" valign="top" rowspan="1" colspan="1">Overall</td><td align="center" valign="top" rowspan="1" colspan="1">5944</td><td align="center" valign="top" rowspan="1" colspan="1">4534 (76.3)</td><td align="center" valign="top" rowspan="1" colspan="1">2833 (47.7)</td><td align="center" valign="top" rowspan="1" colspan="1">814 (13.7)</td><td align="center" valign="top" rowspan="1" colspan="1">5634 (94.8)</td><td align="center" valign="top" rowspan="1" colspan="1">596 (10.6)</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">2016&#x02013;2017</td><td align="center" valign="top" rowspan="1" colspan="1">1427</td><td align="center" valign="top" rowspan="1" colspan="1">1075 (75.3)</td><td align="center" valign="top" rowspan="1" colspan="1">679 (47.6)<sup><xref rid="TFN6" ref-type="table-fn">f</xref></sup></td><td align="center" valign="top" rowspan="1" colspan="1">195 (13.7)<sup><xref rid="TFN6" ref-type="table-fn">f</xref></sup></td><td align="center" valign="top" rowspan="1" colspan="1">1309 (91.7)</td><td align="center" valign="top" rowspan="1" colspan="1">132 (10.1)</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">2017&#x02013;2018</td><td align="center" valign="top" rowspan="1" colspan="1">1362</td><td align="center" valign="top" rowspan="1" colspan="1">1073 (78.8)</td><td align="center" valign="top" rowspan="1" colspan="1">588 (43.2)<sup><xref rid="TFN6" ref-type="table-fn">f</xref></sup></td><td align="center" valign="top" rowspan="1" colspan="1">208 (15.3)<sup><xref rid="TFN6" ref-type="table-fn">f</xref></sup></td><td align="center" valign="top" rowspan="1" colspan="1">1313 (96.4)</td><td align="center" valign="top" rowspan="1" colspan="1">116 (8.8)</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">2018&#x02013;2019</td><td align="center" valign="top" rowspan="1" colspan="1">1679</td><td align="center" valign="top" rowspan="1" colspan="1">1278 (76.1)</td><td align="center" valign="top" rowspan="1" colspan="1">869 (51.8)<sup><xref rid="TFN6" ref-type="table-fn">f</xref></sup></td><td align="center" valign="top" rowspan="1" colspan="1">169 (10.1)<sup><xref rid="TFN6" ref-type="table-fn">f</xref></sup></td><td align="center" valign="top" rowspan="1" colspan="1">1641 (97.7)</td><td align="center" valign="top" rowspan="1" colspan="1">208 (12.7)</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">2019&#x02013;2020</td><td align="center" valign="top" rowspan="1" colspan="1">1476</td><td align="center" valign="top" rowspan="1" colspan="1">1108 (75.1)</td><td align="center" valign="top" rowspan="1" colspan="1">697 (47.2)<sup><xref rid="TFN6" ref-type="table-fn">f</xref></sup></td><td align="center" valign="top" rowspan="1" colspan="1">242 (16.4)<sup><xref rid="TFN6" ref-type="table-fn">f</xref></sup></td><td align="center" valign="top" rowspan="1" colspan="1">1371 (92.9)</td><td align="center" valign="top" rowspan="1" colspan="1">140 (10.2)</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">6 mon-17 years</td><td align="center" valign="top" rowspan="1" colspan="1">1039</td><td align="center" valign="top" rowspan="1" colspan="1">961 (92.5)<sup><xref rid="TFN6" ref-type="table-fn">f</xref></sup></td><td align="center" valign="top" rowspan="1" colspan="1">685 (65.9)<sup><xref rid="TFN6" ref-type="table-fn">f</xref></sup></td><td align="center" valign="top" rowspan="1" colspan="1">110 (10.6)<sup><xref rid="TFN7" ref-type="table-fn">g</xref></sup></td><td align="center" valign="top" rowspan="1" colspan="1">1100 (105.9)</td><td align="center" valign="top" rowspan="1" colspan="1">192 (17.5)</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">18&#x02013;64 years</td><td align="center" valign="top" rowspan="1" colspan="1">3054</td><td align="center" valign="top" rowspan="1" colspan="1">2228 (73.0)<sup><xref rid="TFN6" ref-type="table-fn">f</xref></sup></td><td align="center" valign="top" rowspan="1" colspan="1">1239 (40.6)<sup><xref rid="TFN6" ref-type="table-fn">f</xref></sup></td><td align="center" valign="top" rowspan="1" colspan="1">459 (15.0)<sup><xref rid="TFN7" ref-type="table-fn">g</xref></sup></td><td align="center" valign="top" rowspan="1" colspan="1">2868 (93.9)</td><td align="center" valign="top" rowspan="1" colspan="1">286 (10.0)</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">&#x02265;65 years</td><td align="center" valign="top" rowspan="1" colspan="1">1846</td><td align="center" valign="top" rowspan="1" colspan="1">1345 (72.9)<sup><xref rid="TFN6" ref-type="table-fn">f</xref></sup></td><td align="center" valign="top" rowspan="1" colspan="1">908 (49.2)<sup><xref rid="TFN6" ref-type="table-fn">f</xref></sup></td><td align="center" valign="top" rowspan="1" colspan="1">243 (13.2)<sup><xref rid="TFN7" ref-type="table-fn">g</xref></sup></td><td align="center" valign="top" rowspan="1" colspan="1">1663 (90.1)</td><td align="center" valign="top" rowspan="1" colspan="1">117 (7.0)</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">White race</td><td align="center" valign="top" rowspan="1" colspan="1">4491</td><td align="center" valign="top" rowspan="1" colspan="1">3379 (75.2)<sup><xref rid="TFN6" ref-type="table-fn">f</xref></sup></td><td align="center" valign="top" rowspan="1" colspan="1">2002 (44.6)<sup><xref rid="TFN6" ref-type="table-fn">f</xref></sup></td><td align="center" valign="top" rowspan="1" colspan="1">630 (14.0)</td><td align="center" valign="top" rowspan="1" colspan="1">4179 (93.1)</td><td align="center" valign="top" rowspan="1" colspan="1">322 (7.7)</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Non-white race</td><td align="center" valign="top" rowspan="1" colspan="1">1424</td><td align="center" valign="top" rowspan="1" colspan="1">1133 (79.6)<sup><xref rid="TFN6" ref-type="table-fn">f</xref></sup></td><td align="center" valign="top" rowspan="1" colspan="1">816 (57.3)<sup><xref rid="TFN6" ref-type="table-fn">f</xref></sup></td><td align="center" valign="top" rowspan="1" colspan="1">178 (12.5)</td><td align="center" valign="top" rowspan="1" colspan="1">1425 (100.1)</td><td align="center" valign="top" rowspan="1" colspan="1">266 (18.7)</td></tr></tbody></table><table-wrap-foot><fn id="TFN1"><label>a</label><p id="P41">PA-SIIS = Pennsylvania Statewide Immunization Information System</p></fn><fn id="TFN2"><label>b</label><p id="P42">EMR = Health system electronic medical record</p></fn><fn id="TFN3"><label>c</label><p id="P43">Other sources = Pharmacies, employer records, insurer records,
physician offices, etc.</p></fn><fn id="TFN4"><label>d</label><p id="P44">All sources = PA-SIIS +EMR + Other sources; vaccines recorded in
more than one source are only counted once for All sources.</p></fn><fn id="TFN5"><label>e</label><p id="P45">Discovered vaccination = vaccines found in various sources among
those who self-reported no vaccination. .%=discovered/all sources</p></fn><fn id="TFN6"><label>f</label><p id="P46"><italic>P</italic>&#x02264;0.001 for differences across groups</p></fn><fn id="TFN7"><label>g</label><p id="P47"><italic>P</italic>&#x02264;0.01 for differences across groups</p></fn></table-wrap-foot></table-wrap><table-wrap id="T3" position="float" orientation="landscape"><label>Table 3.</label><caption><p id="P48">Agreement between self-report (gold standard) and alternate sources for
influenza vaccination status, N=9930</p></caption><table frame="void" rules="none"><colgroup span="1"><col align="left" valign="middle" span="1"/><col align="left" valign="middle" span="1"/><col align="left" valign="middle" span="1"/><col align="left" valign="middle" span="1"/><col align="left" valign="middle" span="1"/><col align="left" valign="middle" span="1"/><col align="left" valign="middle" span="1"/><col align="left" valign="middle" span="1"/></colgroup><thead><tr><th align="left" valign="top" rowspan="1" colspan="1">EMR</th><th align="left" valign="top" rowspan="1" colspan="1"/><th align="left" valign="top" rowspan="1" colspan="1"/><th align="left" valign="top" rowspan="1" colspan="1"/><th align="left" valign="top" rowspan="1" colspan="1"/><th align="left" valign="top" rowspan="1" colspan="1"/><th align="left" valign="top" rowspan="1" colspan="1"/><th align="left" valign="top" rowspan="1" colspan="1"/></tr><tr><th align="left" valign="top" rowspan="1" colspan="1">&#x02003;Vaccinations reported in each
source</th><th align="center" valign="top" rowspan="1" colspan="1">Agreement</th><th align="center" valign="top" rowspan="1" colspan="1">Kappa (95% CI)</th><th align="center" valign="top" rowspan="1" colspan="1">PPV, % (95% CI)</th><th align="center" valign="top" rowspan="1" colspan="1">NPV, % (95% CI)</th><th align="center" valign="top" rowspan="1" colspan="1">Sensitivity, % (95% CI)</th><th align="center" valign="top" rowspan="1" colspan="1">Specificity, % (95% CI)</th><th align="center" valign="top" rowspan="1" colspan="1">Area under ROC curve</th></tr></thead><tbody><tr><td align="left" valign="middle" rowspan="1" colspan="1">&#x02003;<graphic xlink:href="nihms-1754034-t0004"/><break/>&#x02003;Unvaccinated =
3641</td><td align="center" valign="middle" rowspan="1" colspan="1">62%</td><td align="center" valign="middle" rowspan="1" colspan="1">0.29 (.28, .31)</td><td align="center" valign="middle" rowspan="1" colspan="1">88 (87, 89)</td><td align="center" valign="middle" rowspan="1" colspan="1">51 (50, 52)</td><td align="center" valign="middle" rowspan="1" colspan="1">42 (41, 43)</td><td align="center" valign="middle" rowspan="1" colspan="1">91 (90, 92)</td><td align="center" valign="top" rowspan="1" colspan="1">67%</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">
<hr/>
</td><td align="left" valign="top" colspan="7" rowspan="1"/></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">
<bold>PA-SIIS</bold>
</td><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/></tr><tr><td align="left" valign="middle" rowspan="1" colspan="1">&#x02003;<graphic xlink:href="nihms-1754034-t0005"/><break/>&#x02003;Unvaccinated =
3525</td><td align="center" valign="middle" rowspan="1" colspan="1">77%</td><td align="center" valign="middle" rowspan="1" colspan="1">0.54 (.52, .55)</td><td align="center" valign="middle" rowspan="1" colspan="1">90 (89, 91)</td><td align="center" valign="middle" rowspan="1" colspan="1">65 (64, 67)</td><td align="center" valign="middle" rowspan="1" colspan="1">69 (67, 70)</td><td align="center" valign="middle" rowspan="1" colspan="1">88 (87, 89)</td><td align="center" valign="middle" rowspan="1" colspan="1">78%</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">
<bold>All other sources</bold>
</td><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/></tr><tr><td align="left" valign="middle" rowspan="1" colspan="1">&#x02003;<graphic xlink:href="nihms-1754034-t0006"/><break/>&#x02003;Unvaccinated =
3390</td><td align="center" valign="middle" rowspan="1" colspan="1">85%</td><td align="center" valign="middle" rowspan="1" colspan="1">0.69 (.67, .70)</td><td align="center" valign="middle" rowspan="1" colspan="1">89 (88, 90)</td><td align="center" valign="middle" rowspan="1" colspan="1">79 (78, 80))</td><td align="center" valign="middle" rowspan="1" colspan="1">85 (84, 86)</td><td align="center" valign="middle" rowspan="1" colspan="1">85 (84, 86)</td><td align="center" valign="middle" rowspan="1" colspan="1">85%</td></tr></tbody></table></table-wrap><table-wrap id="T4" position="float" orientation="landscape"><label>Table 4.</label><caption><p id="P49">Agreement between PA-SIIS (gold standard) and alternate sources for
influenza vaccination status, N=9930</p></caption><table frame="void" rules="none"><colgroup span="1"><col align="left" valign="middle" span="1"/><col align="left" valign="middle" span="1"/><col align="left" valign="middle" span="1"/><col align="left" valign="middle" span="1"/><col align="left" valign="middle" span="1"/><col align="left" valign="middle" span="1"/><col align="left" valign="middle" span="1"/><col align="left" valign="middle" span="1"/></colgroup><thead><tr><th align="left" valign="top" rowspan="1" colspan="1">EMR</th><th align="left" valign="top" rowspan="1" colspan="1"/><th align="left" valign="top" rowspan="1" colspan="1"/><th align="left" valign="top" rowspan="1" colspan="1"/><th align="left" valign="top" rowspan="1" colspan="1"/><th align="left" valign="top" rowspan="1" colspan="1"/><th align="left" valign="top" rowspan="1" colspan="1"/><th align="left" valign="top" rowspan="1" colspan="1"/></tr><tr><th align="left" valign="top" rowspan="1" colspan="1">&#x02003;Vaccines reported in each source</th><th align="center" valign="top" rowspan="1" colspan="1">Agreement</th><th align="center" valign="top" rowspan="1" colspan="1">Kappa (95% CI)</th><th align="center" valign="top" rowspan="1" colspan="1">PPV, % (95% CI)</th><th align="center" valign="top" rowspan="1" colspan="1">NPV, % (95% CI)</th><th align="center" valign="top" rowspan="1" colspan="1">Sensitivity, % (95% CI)</th><th align="center" valign="top" rowspan="1" colspan="1">Specificity, % (95% CI)</th><th align="center" valign="top" rowspan="1" colspan="1">Area under ROC curve</th></tr></thead><tbody><tr><td align="left" valign="middle" rowspan="1" colspan="1">&#x02003;<graphic xlink:href="nihms-1754034-t0007"/><break/>&#x02003;Unvaccinated =
5110</td><td align="center" valign="middle" rowspan="1" colspan="1">77%</td><td align="center" valign="middle" rowspan="1" colspan="1">0.52 (.51, .54)</td><td align="center" valign="middle" rowspan="1" colspan="1">90 (89, 91)</td><td align="center" valign="middle" rowspan="1" colspan="1">72 (71, 73)</td><td align="center" valign="middle" rowspan="1" colspan="1">56 (55, 58)</td><td align="center" valign="middle" rowspan="1" colspan="1">95 (94, 95.3)</td><td align="center" valign="middle" rowspan="1" colspan="1">75%</td></tr><tr><td align="left" valign="top" colspan="8" rowspan="1">
<hr/>
</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">
<bold>Self-report, EMR, and other
sources</bold>
</td><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/></tr><tr><td align="left" valign="middle" rowspan="1" colspan="1">&#x02003;<graphic xlink:href="nihms-1754034-t0008"/><break/>&#x02003;Unvaccinated =
3390</td><td align="center" valign="middle" rowspan="1" colspan="1">78%</td><td align="center" valign="middle" rowspan="1" colspan="1">0.57 (.56, .58)</td><td align="center" valign="middle" rowspan="1" colspan="1">68 (67, 70)</td><td align="center" valign="middle" rowspan="1" colspan="1">95 (94, 96)</td><td align="center" valign="middle" rowspan="1" colspan="1">96 (95, 97</td><td align="center" valign="middle" rowspan="1" colspan="1">63 (62, 64)</td><td align="center" valign="middle" rowspan="1" colspan="1">79%</td></tr></tbody></table></table-wrap><table-wrap id="T5" position="float" orientation="landscape"><label>Table 5.</label><caption><p id="P50">Completeness of bidirectional interface between PA-SIIS and EMR</p></caption><table frame="box" rules="none"><colgroup span="1"><col align="left" valign="middle" span="1"/><col align="left" valign="middle" span="1"/><col align="left" valign="middle" span="1"/><col align="left" valign="middle" span="1"/><col align="left" valign="middle" span="1"/><col align="left" valign="middle" span="1"/></colgroup><thead><tr><th align="left" valign="top" rowspan="1" colspan="1"/><th align="center" valign="top" colspan="3" rowspan="1">Vaccination recorded in:</th><th align="center" valign="top" colspan="2" rowspan="1">Vaccinations sent from:</th></tr><tr><th align="left" valign="top" rowspan="1" colspan="1"/><th align="center" valign="top" rowspan="1" colspan="1">Both PA-SIIS and EMR</th><th align="center" valign="top" rowspan="1" colspan="1">PA-SIIS only</th><th align="center" valign="top" rowspan="1" colspan="1">EMR only</th><th align="center" valign="top" rowspan="1" colspan="1">PA-SIIS to EMR</th><th align="center" valign="top" rowspan="1" colspan="1">EMR to PA-SIIS</th></tr><tr><th align="left" valign="top" rowspan="1" colspan="1"/><th align="center" valign="top" rowspan="1" colspan="1">(a)</th><th align="center" valign="top" rowspan="1" colspan="1">(b)</th><th align="center" valign="top" rowspan="1" colspan="1">(c)</th><th align="center" valign="top" rowspan="1" colspan="1">(a/b*100)</th><th align="center" valign="top" rowspan="1" colspan="1">(a/c*100)</th></tr><tr><th align="left" valign="top" colspan="6" rowspan="1">
<hr/>
</th></tr></thead><tbody><tr><td align="left" valign="top" rowspan="1" colspan="1">Overall</td><td align="center" valign="top" rowspan="1" colspan="1">2547</td><td align="center" valign="top" rowspan="1" colspan="1">4534</td><td align="center" valign="top" rowspan="1" colspan="1">2833</td><td align="center" valign="top" rowspan="1" colspan="1">56.2%</td><td align="center" valign="top" rowspan="1" colspan="1">89.9%</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Season</td><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">&#x02003;2016&#x02013;2017</td><td align="center" valign="top" rowspan="1" colspan="1">640</td><td align="center" valign="top" rowspan="1" colspan="1">1075</td><td align="center" valign="top" rowspan="1" colspan="1">679</td><td align="center" valign="top" rowspan="1" colspan="1">59.5%</td><td align="center" valign="top" rowspan="1" colspan="1">94.3%</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">&#x02003;2017&#x02013;2018</td><td align="center" valign="top" rowspan="1" colspan="1">556</td><td align="center" valign="top" rowspan="1" colspan="1">1073</td><td align="center" valign="top" rowspan="1" colspan="1">588</td><td align="center" valign="top" rowspan="1" colspan="1">51.8%</td><td align="center" valign="top" rowspan="1" colspan="1">94.6%</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">&#x02003;2018&#x02013;2019</td><td align="center" valign="top" rowspan="1" colspan="1">675</td><td align="center" valign="top" rowspan="1" colspan="1">1278</td><td align="center" valign="top" rowspan="1" colspan="1">869</td><td align="center" valign="top" rowspan="1" colspan="1">52.8%</td><td align="center" valign="top" rowspan="1" colspan="1">77.7%</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">&#x02003;2019&#x02013;2020</td><td align="center" valign="top" rowspan="1" colspan="1">676</td><td align="center" valign="top" rowspan="1" colspan="1">1108</td><td align="center" valign="top" rowspan="1" colspan="1">697</td><td align="center" valign="top" rowspan="1" colspan="1">61.0%</td><td align="center" valign="top" rowspan="1" colspan="1">97.0%</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Age group</td><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">&#x02003;6m-17 years</td><td align="center" valign="top" rowspan="1" colspan="1">656</td><td align="center" valign="top" rowspan="1" colspan="1">961</td><td align="center" valign="top" rowspan="1" colspan="1">685</td><td align="center" valign="top" rowspan="1" colspan="1">68.3%</td><td align="center" valign="top" rowspan="1" colspan="1">95.8%</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">&#x02003;18&#x02013;64 years</td><td align="center" valign="top" rowspan="1" colspan="1">1058</td><td align="center" valign="top" rowspan="1" colspan="1">2228</td><td align="center" valign="top" rowspan="1" colspan="1">1239</td><td align="center" valign="top" rowspan="1" colspan="1">47.5%</td><td align="center" valign="top" rowspan="1" colspan="1">85.4%</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">&#x02003;&#x0003e;= 65 years</td><td align="center" valign="top" rowspan="1" colspan="1">833</td><td align="center" valign="top" rowspan="1" colspan="1">1345</td><td align="center" valign="top" rowspan="1" colspan="1">908</td><td align="center" valign="top" rowspan="1" colspan="1">61.9%</td><td align="center" valign="top" rowspan="1" colspan="1">91.7%</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">Race</td><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/><td align="left" valign="top" rowspan="1" colspan="1"/></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">&#x02003;White</td><td align="center" valign="top" rowspan="1" colspan="1">1832</td><td align="center" valign="top" rowspan="1" colspan="1">3379</td><td align="center" valign="top" rowspan="1" colspan="1">2002</td><td align="center" valign="top" rowspan="1" colspan="1">54.2%</td><td align="center" valign="top" rowspan="1" colspan="1">91.5%</td></tr><tr><td align="left" valign="top" rowspan="1" colspan="1">&#x02003;Non-White</td><td align="center" valign="top" rowspan="1" colspan="1">702</td><td align="center" valign="top" rowspan="1" colspan="1">1133</td><td align="center" valign="top" rowspan="1" colspan="1">816</td><td align="center" valign="top" rowspan="1" colspan="1">62.0%</td><td align="center" valign="top" rowspan="1" colspan="1">86.0%</td></tr></tbody></table></table-wrap></floats-group></article>