<?xml version="1.0" encoding="UTF-8"?><?xml-stylesheet type="text/xsl" href="static/style.xsl"?><OAI-PMH xmlns="http://www.openarchives.org/OAI/2.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd"><responseDate>2026-11-22T21:40:09Z</responseDate><request verb="GetRecord" identifier="oai:uwspace.uwaterloo.ca:10012/18622" metadataPrefix="dim">https://dspacemainprd01.lib.uwaterloo.ca/server/oai/request</request><GetRecord><record><header><identifier>oai:uwspace.uwaterloo.ca:10012/18622</identifier><datestamp>2025-03-26T23:32:35Z</datestamp><setSpec>com_10012_9924</setSpec><setSpec>com_10012_1</setSpec><setSpec>col_10012_9930</setSpec><setSpec>col_10012_6</setSpec></header><metadata><dim:dim xmlns:dim="http://www.dspace.org/xmlns/dspace/dim" xmlns:doc="http://www.lyncode.com/xoai" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.dspace.org/xmlns/dspace/dim http://www.dspace.org/schema/dim.xsd">
   <dim:field mdschema="dc" element="contributor" qualifier="advisor">Grossman, Maura</dim:field>
   <dim:field mdschema="dc" element="contributor" qualifier="author" authority="65072e11-cef8-45c5-aca5-fc6737f22e79" confidence="600">Alagappan, Solaiappan</dim:field>
   <dim:field mdschema="dc" element="date" qualifier="accessioned">2022-08-23T18:08:12Z</dim:field>
   <dim:field mdschema="dc" element="date" qualifier="available">2022-08-23T18:08:12Z</dim:field>
   <dim:field mdschema="dc" element="date" qualifier="issued">2022-08-23</dim:field>
   <dim:field mdschema="dc" element="date" qualifier="submitted">2022-08-12</dim:field>
   <dim:field mdschema="dc" element="identifier" qualifier="uri">http://hdl.handle.net/10012/18622</dim:field>
   <dim:field mdschema="dc" element="description" qualifier="abstract" lang="en">Electronic Discovery (eDiscovery), a use-case of High-Recall Information Retrieval&#xd;
(HRIR), seeks to obtain substantially all and only the relevant documents responsive to a&#xd;
request for production in litigation. Applications of HRIR typically use a human as their&#xd;
oracle to determine the relevance for a large number of documents, which is expensive both&#xd;
in terms of time/effort and cost. HRIR experts suggest that Continuous Active Learning&#xd;
(CAL) systems, the state-of-the-art information retrieval (IR) tools used for eDiscovery&#xd;
have the potential to achieve superior results and achieving them is limited primarily by&#xd;
the fallibility of the accuracy of human relevance assessments.&#xd;
&#xd;
In this research, we seek to understand the impact of the error rate in human relevance&#xd;
feedback on CAL systems and attempt to address them using six distinct multi-user–&#xd;
based, hybrid, human-computer assessment strategies. In contrast to the widely used&#xd;
single-user-based, hybrid, human-computer assessment strategy, these multi-user strategies&#xd;
re-provision resources to re-reviewing documents that the user may have misjudged, rather&#xd;
than examining more documents, in the pursuit of mitigating human relevance feedback&#xd;
error, while also achieving a high-recall and high-precision review. Within the constraints&#xd;
of a specified review budget, we want to determine which review strategy has the best&#xd;
chance of precisely retrieving more relevant documents.&#xd;
&#xd;
Our results show that leveraging a multi-user review strategy that “efficiently” uses&#xd;
three reviewers to review documents (CAL QC–Type 1) and a multi-user review strategy&#xd;
that uses the CAL system as one of the users in a three-reviewer approach (CAL QC–Type&#xd;
2) can enable the end-to-end CAL system to achieve a significantly higher recall and higher&#xd;
precision when compared to that achieved by a single-user-based review strategy while&#xd;
employing the same review budget. This research provides evidence that CAL systems have&#xd;
the potential to better accommodate the needs of the HRIR applications by incorporating&#xd;
multi-user review strategies.</dim:field>
   <dim:field mdschema="dc" element="language" qualifier="iso" lang="en">en</dim:field>
   <dim:field mdschema="dc" element="publisher" lang="en">University of Waterloo</dim:field>
   <dim:field mdschema="dc" element="title" lang="en">Determining the Effectiveness of Multi-user, Hybrid, Human-Computer Assessments for High Recall Information Retrieval Systems</dim:field>
   <dim:field mdschema="dc" element="type" lang="en">Master Thesis</dim:field>
   <dim:field mdschema="dc" element="pending">false</dim:field>
   <dim:field mdschema="uws-etd" element="degree" qualifier="department" lang="en">David R. Cheriton School of Computer Science</dim:field>
   <dim:field mdschema="uws-etd" element="degree" qualifier="discipline" lang="en">Computer Science</dim:field>
   <dim:field mdschema="uws-etd" element="degree" qualifier="grantor" lang="en">University of Waterloo</dim:field>
   <dim:field mdschema="uws-etd" element="degree" lang="en">Master of Mathematics</dim:field>
   <dim:field mdschema="uws-etd" element="embargo" qualifier="terms" lang="en">0</dim:field>
   <dim:field mdschema="uws" element="contributor" qualifier="advisor">Grossman, Maura</dim:field>
   <dim:field mdschema="uws" element="contributor" qualifier="affiliation1" lang="en">Faculty of Mathematics</dim:field>
   <dim:field mdschema="uws" element="published" qualifier="city" lang="en">Waterloo</dim:field>
   <dim:field mdschema="uws" element="published" qualifier="country" lang="en">Canada</dim:field>
   <dim:field mdschema="uws" element="published" qualifier="province" lang="en">Ontario</dim:field>
   <dim:field mdschema="uws" element="typeOfResource" lang="en">Text</dim:field>
   <dim:field mdschema="uws" element="peerReviewStatus" lang="en">Unreviewed</dim:field>
   <dim:field mdschema="uws" element="scholarLevel" lang="en">Graduate</dim:field>
   <dim:field mdschema="others" element="access-status">open.access</dim:field>
</dim:dim>
</metadata></record></GetRecord></OAI-PMH>