<?xml version="1.0" encoding="UTF-8"?><?xml-stylesheet type="text/xsl" href="static/style.xsl"?><OAI-PMH xmlns="http://www.openarchives.org/OAI/2.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd"><responseDate>2026-11-22T21:04:03Z</responseDate><request verb="GetRecord" identifier="oai:uwspace.uwaterloo.ca:10012/18586" metadataPrefix="dim">https://dspacemainprd01.lib.uwaterloo.ca/server/oai/request</request><GetRecord><record><header><identifier>oai:uwspace.uwaterloo.ca:10012/18586</identifier><datestamp>2025-03-26T23:32:37Z</datestamp><setSpec>com_10012_9900</setSpec><setSpec>com_10012_1</setSpec><setSpec>col_10012_9908</setSpec><setSpec>col_10012_6</setSpec></header><metadata><dim:dim xmlns:dim="http://www.dspace.org/xmlns/dspace/dim" xmlns:doc="http://www.lyncode.com/xoai" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.dspace.org/xmlns/dspace/dim http://www.dspace.org/schema/dim.xsd">
   <dim:field mdschema="dc" element="contributor" qualifier="advisor">Karray, Fakhri</dim:field>
   <dim:field mdschema="dc" element="contributor" qualifier="author" authority="2c097dca-7dc9-490e-9a6c-c67319a9bd29" confidence="600">Xu, Ruizhou</dim:field>
   <dim:field mdschema="dc" element="date" qualifier="accessioned">2022-08-19T17:39:57Z</dim:field>
   <dim:field mdschema="dc" element="date" qualifier="available">2022-08-19T17:39:57Z</dim:field>
   <dim:field mdschema="dc" element="date" qualifier="issued">2022-08-19</dim:field>
   <dim:field mdschema="dc" element="date" qualifier="submitted">2022-08-10</dim:field>
   <dim:field mdschema="dc" element="identifier" qualifier="uri">http://hdl.handle.net/10012/18586</dim:field>
   <dim:field mdschema="dc" element="description" qualifier="abstract" lang="en">Neural networks are powerful solutions to help with decision making and solve complex&#xd;
problems in recent years. In the domain of natural language processing, BERT and&#xd;
its variants significantly outperform other network structures. It learns general linguistic&#xd;
knowledge from a large corpus in the pre-training stage and utilizes that to solve downstream&#xd;
tasks. However, the size of this kind of model is enormous and causes the issue&#xd;
of overparameterization. This makes the model deployment on small edge devices less&#xd;
scalable and flexible.&#xd;
In this thesis, we study how to compress the BERT model in a structured pruning&#xd;
manner. We proposed the neural slimming technique to assess the importance of each&#xd;
neuron and designed the cost function and pruning strategy to remove neurons that make&#xd;
zero or less contribution to the prediction. After getting fine-tuned on the downstream&#xd;
tasks, the model can learn a more compact structure, and we name it SlimBERT.&#xd;
We tested our method on 7 GLUE tasks and used only 10% original parameters to recover 94% of the original performance. It also reduced the run-time memory and increased the&#xd;
inference speed at the same time. Compared to knowledge distillation methods and other&#xd;
structured pruning methods, the proposed approach achieved better performance under&#xd;
different metrics with the same compression ratio. Moreover, our method also improved&#xd;
the interpretability of BERT. By analyzing neurons with a significant contribution, we can&#xd;
observe that BERT utilizes different components and subnetworks according to different&#xd;
tasks.</dim:field>
   <dim:field mdschema="dc" element="language" qualifier="iso" lang="en">en</dim:field>
   <dim:field mdschema="dc" element="publisher" lang="en">University of Waterloo</dim:field>
   <dim:field mdschema="dc" element="relation" qualifier="uri" lang="en">https://gluebenchmark.com/</dim:field>
   <dim:field mdschema="dc" element="subject" lang="en">model compression</dim:field>
   <dim:field mdschema="dc" element="subject" lang="en">NLP</dim:field>
   <dim:field mdschema="dc" element="subject" lang="en">pre-trained language model</dim:field>
   <dim:field mdschema="dc" element="subject" lang="en">structured pruning</dim:field>
   <dim:field mdschema="dc" element="subject" lang="en">BERT</dim:field>
   <dim:field mdschema="dc" element="title" lang="en">Compression and Analysis of Pre-trained Language Model using Neural Slimming</dim:field>
   <dim:field mdschema="dc" element="type" lang="en">Master Thesis</dim:field>
   <dim:field mdschema="dc" element="pending">false</dim:field>
   <dim:field mdschema="uws-etd" element="degree" qualifier="department" lang="en">Electrical and Computer Engineering</dim:field>
   <dim:field mdschema="uws-etd" element="degree" qualifier="discipline" lang="en">Electrical and Computer Engineering</dim:field>
   <dim:field mdschema="uws-etd" element="degree" qualifier="grantor" lang="en">University of Waterloo</dim:field>
   <dim:field mdschema="uws-etd" element="degree" lang="en">Master of Applied Science</dim:field>
   <dim:field mdschema="uws-etd" element="embargo" qualifier="terms" lang="en">0</dim:field>
   <dim:field mdschema="uws" element="contributor" qualifier="advisor">Karray, Fakhri</dim:field>
   <dim:field mdschema="uws" element="contributor" qualifier="affiliation1" lang="en">Faculty of Engineering</dim:field>
   <dim:field mdschema="uws" element="published" qualifier="city" lang="en">Waterloo</dim:field>
   <dim:field mdschema="uws" element="published" qualifier="country" lang="en">Canada</dim:field>
   <dim:field mdschema="uws" element="published" qualifier="province" lang="en">Ontario</dim:field>
   <dim:field mdschema="uws" element="typeOfResource" lang="en">Text</dim:field>
   <dim:field mdschema="uws" element="peerReviewStatus" lang="en">Unreviewed</dim:field>
   <dim:field mdschema="uws" element="scholarLevel" lang="en">Graduate</dim:field>
   <dim:field mdschema="others" element="access-status">open.access</dim:field>
</dim:dim>
</metadata></record></GetRecord></OAI-PMH>