@inproceedings{wang-mcallester-2020-fly,
title = "On-The-Fly Information Retrieval Augmentation for Language Models",
author = "Wang, Hai and
McAllester, David",
editor = "Bonial, Claire and
Caselli, Tommaso and
Chaturvedi, Snigdha and
Clark, Elizabeth and
Huang, Ruihong and
Iyyer, Mohit and
Jaimes, Alejandro and
Ji, Heng and
Martin, Lara J. and
Miller, Ben and
Mitamura, Teruko and
Peng, Nanyun and
Tetreault, Joel",
booktitle = "Proceedings of the First Joint Workshop on Narrative Understanding, Storylines, and Events",
month = jul,
year = "2020",
address = "Online",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2020.nuse-1.14",
doi = "10.18653/v1/2020.nuse-1.14",
pages = "114--119",
abstract = "Here we experiment with the use of information retrieval as an augmentation for pre-trained language models. The text corpus used in information retrieval can be viewed as form of episodic memory which grows over time. By augmenting GPT 2.0 with information retrieval we achieve a zero shot 15{\%} relative reduction in perplexity on Gigaword corpus without any re-training. We also validate our IR augmentation on an event co-reference task.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="wang-mcallester-2020-fly">
<titleInfo>
<title>On-The-Fly Information Retrieval Augmentation for Language Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Hai</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">David</namePart>
<namePart type="family">McAllester</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2020-07</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the First Joint Workshop on Narrative Understanding, Storylines, and Events</title>
</titleInfo>
<name type="personal">
<namePart type="given">Claire</namePart>
<namePart type="family">Bonial</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tommaso</namePart>
<namePart type="family">Caselli</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Snigdha</namePart>
<namePart type="family">Chaturvedi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Elizabeth</namePart>
<namePart type="family">Clark</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ruihong</namePart>
<namePart type="family">Huang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mohit</namePart>
<namePart type="family">Iyyer</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Alejandro</namePart>
<namePart type="family">Jaimes</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Heng</namePart>
<namePart type="family">Ji</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Lara</namePart>
<namePart type="given">J</namePart>
<namePart type="family">Martin</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ben</namePart>
<namePart type="family">Miller</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Teruko</namePart>
<namePart type="family">Mitamura</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Nanyun</namePart>
<namePart type="family">Peng</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Joel</namePart>
<namePart type="family">Tetreault</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Online</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>Here we experiment with the use of information retrieval as an augmentation for pre-trained language models. The text corpus used in information retrieval can be viewed as form of episodic memory which grows over time. By augmenting GPT 2.0 with information retrieval we achieve a zero shot 15% relative reduction in perplexity on Gigaword corpus without any re-training. We also validate our IR augmentation on an event co-reference task.</abstract>
<identifier type="citekey">wang-mcallester-2020-fly</identifier>
<identifier type="doi">10.18653/v1/2020.nuse-1.14</identifier>
<location>
<url>https://aclanthology.org/2020.nuse-1.14</url>
</location>
<part>
<date>2020-07</date>
<extent unit="page">
<start>114</start>
<end>119</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T On-The-Fly Information Retrieval Augmentation for Language Models
%A Wang, Hai
%A McAllester, David
%Y Bonial, Claire
%Y Caselli, Tommaso
%Y Chaturvedi, Snigdha
%Y Clark, Elizabeth
%Y Huang, Ruihong
%Y Iyyer, Mohit
%Y Jaimes, Alejandro
%Y Ji, Heng
%Y Martin, Lara J.
%Y Miller, Ben
%Y Mitamura, Teruko
%Y Peng, Nanyun
%Y Tetreault, Joel
%S Proceedings of the First Joint Workshop on Narrative Understanding, Storylines, and Events
%D 2020
%8 July
%I Association for Computational Linguistics
%C Online
%F wang-mcallester-2020-fly
%X Here we experiment with the use of information retrieval as an augmentation for pre-trained language models. The text corpus used in information retrieval can be viewed as form of episodic memory which grows over time. By augmenting GPT 2.0 with information retrieval we achieve a zero shot 15% relative reduction in perplexity on Gigaword corpus without any re-training. We also validate our IR augmentation on an event co-reference task.
%R 10.18653/v1/2020.nuse-1.14
%U https://aclanthology.org/2020.nuse-1.14
%U https://doi.org/10.18653/v1/2020.nuse-1.14
%P 114-119
Markdown (Informal)
[On-The-Fly Information Retrieval Augmentation for Language Models](https://aclanthology.org/2020.nuse-1.14) (Wang & McAllester, NUSE-WNU 2020)
ACL