<?xml version="1.0" encoding="UTF-8"?><?xml-stylesheet type="text/xsl" href="static/style.xsl"?><OAI-PMH xmlns="http://www.openarchives.org/OAI/2.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd"><responseDate>2026-11-22T11:46:28Z</responseDate><request verb="GetRecord" identifier="oai:uwspace.uwaterloo.ca:10012/20847" metadataPrefix="dim">https://dspacemainprd01.lib.uwaterloo.ca/server/oai/request</request><GetRecord><record><header><identifier>oai:uwspace.uwaterloo.ca:10012/20847</identifier><datestamp>2025-03-26T23:37:51Z</datestamp><setSpec>com_10012_9924</setSpec><setSpec>com_10012_1</setSpec><setSpec>col_10012_9930</setSpec><setSpec>col_10012_6</setSpec></header><metadata><dim:dim xmlns:dim="http://www.dspace.org/xmlns/dspace/dim" xmlns:doc="http://www.lyncode.com/xoai" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.dspace.org/xmlns/dspace/dim http://www.dspace.org/schema/dim.xsd">
   <dim:field mdschema="dc" element="contributor" qualifier="advisor">Brown, Dan</dim:field>
   <dim:field mdschema="dc" element="contributor" qualifier="author">Khatun, Aisha</dim:field>
   <dim:field mdschema="dc" element="date" qualifier="accessioned">2024-08-22T14:12:00Z</dim:field>
   <dim:field mdschema="dc" element="date" qualifier="available">2024-08-22T14:12:00Z</dim:field>
   <dim:field mdschema="dc" element="date" qualifier="issued">2024-08-22</dim:field>
   <dim:field mdschema="dc" element="date" qualifier="submitted">2024-08-14</dim:field>
   <dim:field mdschema="dc" element="identifier" qualifier="uri">https://hdl.handle.net/10012/20847</dim:field>
   <dim:field mdschema="dc" element="description" qualifier="abstract">Large Language Models (LLMs) have rapidly advanced, becoming general-purpose assistants and creative partners. Despite their widespread use, LLMs exhibit significant vulnerabilities to prompt variations and struggle with task understanding, leading to inconsistencies and factual inaccuracies in their responses. Traditional Natural Language Processing (NLP) benchmarks often overlook nuances in LLM behavior and reliability. This thesis addresses this gap by curating a dataset across six categories: Fact, Conspiracy, Controversy, Misconception, Stereotype, and Fiction. We rigorously define LLMs&amp;apos; factual accuracy, consistency, and robustness to prompt variations using diverse response formats and question variations, and evaluate these on 37 models. Our findings reveal LLMs&amp;apos; volatility and unreliability, particularly in the Controversy and Misconception categories, where conflicting training data impedes performance. Additionally, we explore LLMs&amp;apos; ability to generate coherent fictional narratives, probing their ability to retain and effectively utilize factual information, a critical requirement for creative tasks like story generation. While LLMs offer versatile applications, their reliability hinges on addressing challenges in prompt understanding and response consistency, emphasizing the need for ongoing research to enhance their performance across diverse tasks and applications.</dim:field>
   <dim:field mdschema="dc" element="language" qualifier="iso">en</dim:field>
   <dim:field mdschema="dc" element="publisher" lang="en">University of Waterloo</dim:field>
   <dim:field mdschema="dc" element="relation" qualifier="uri">https://borealisdata.ca/dataset.xhtml?persistentId=doi:10.5683/SP3/5MZWBV</dim:field>
   <dim:field mdschema="dc" element="relation" qualifier="uri">https://github.com/tanny411/llm-reliability-and-consistency-evaluation</dim:field>
   <dim:field mdschema="dc" element="subject">Large Language Model</dim:field>
   <dim:field mdschema="dc" element="subject">Computational Creativity</dim:field>
   <dim:field mdschema="dc" element="subject">Story Generation</dim:field>
   <dim:field mdschema="dc" element="subject">Consistency</dim:field>
   <dim:field mdschema="dc" element="subject">Robustness</dim:field>
   <dim:field mdschema="dc" element="subject">LLM</dim:field>
   <dim:field mdschema="dc" element="subject">AI</dim:field>
   <dim:field mdschema="dc" element="subject">GPT 3</dim:field>
   <dim:field mdschema="dc" element="subject">GPT 4</dim:field>
   <dim:field mdschema="dc" element="subject">MCQ</dim:field>
   <dim:field mdschema="dc" element="subject">Multiple Choice Question</dim:field>
   <dim:field mdschema="dc" element="subject">Dataset</dim:field>
   <dim:field mdschema="dc" element="subject">Factual Accuracy</dim:field>
   <dim:field mdschema="dc" element="subject">Artificial Intelligence</dim:field>
   <dim:field mdschema="dc" element="subject">NLP</dim:field>
   <dim:field mdschema="dc" element="subject">Natural Language Processing</dim:field>
   <dim:field mdschema="dc" element="title">Uncovering the Reliability and Consistency of AI Language Models: A Systematic Study</dim:field>
   <dim:field mdschema="dc" element="type">Master Thesis</dim:field>
   <dim:field mdschema="dc" element="pending">false</dim:field>
   <dim:field mdschema="uws-etd" element="degree" qualifier="department">David R. Cheriton School of Computer Science</dim:field>
   <dim:field mdschema="uws-etd" element="degree" qualifier="discipline">Computer Science</dim:field>
   <dim:field mdschema="uws-etd" element="degree" qualifier="grantor" lang="en">University of Waterloo</dim:field>
   <dim:field mdschema="uws-etd" element="degree">Master of Mathematics</dim:field>
   <dim:field mdschema="uws-etd" element="embargo" qualifier="terms">0</dim:field>
   <dim:field mdschema="uws" element="contributor" qualifier="advisor">Brown, Dan</dim:field>
   <dim:field mdschema="uws" element="contributor" qualifier="affiliation1">Faculty of Mathematics</dim:field>
   <dim:field mdschema="uws" element="published" qualifier="city" lang="en">Waterloo</dim:field>
   <dim:field mdschema="uws" element="published" qualifier="country" lang="en">Canada</dim:field>
   <dim:field mdschema="uws" element="published" qualifier="province" lang="en">Ontario</dim:field>
   <dim:field mdschema="uws" element="typeOfResource" lang="en">Text</dim:field>
   <dim:field mdschema="uws" element="peerReviewStatus" lang="en">Unreviewed</dim:field>
   <dim:field mdschema="uws" element="scholarLevel" lang="en">Graduate</dim:field>
   <dim:field mdschema="others" element="access-status">open.access</dim:field>
</dim:dim>
</metadata></record></GetRecord></OAI-PMH>