This was found by enabling the float-cast-overflow check in UBSan and fuzzing. This type of issue can create inconsistencies across platforms, architectures and optimization levels.

Found with m-c 20220304-ee4f4beb8186.

This issue is triggered easily by fuzzers and will be hit frequently once float-cast-overflow is enabled by default.

To enable this check add the following to your mozconfig:

ac_add_options --enable-undefined-sanitizer="float-cast-overflow"

src/swgl_ext.h:537:16: runtime error: -nan is outside the range of representable values of type 'int'
    #0 0x7fc5c61ffea6 in int spanNeedsScale<glsl::vec2>(int, glsl::vec2) /home/twsmith/code/mozilla-central/gfx/wr/swgl/src/swgl_ext.h:537:16
    #1 0x7fc5c61ffea6 in LinearFilter needsTextureLinear<glsl::sampler2D_impl*, glsl::vec2>(glsl::sampler2D_impl*, glsl::vec2, int) /home/twsmith/code/mozilla-central/gfx/wr/swgl/src/swgl_ext.h:550:19
    #2 0x7fc5c61ffea6 in int blendTextureLinearRepeat<true, glsl::sampler2D_impl*, NoColor, unsigned int>(glsl::sampler2D_impl*, glsl::vec2, int, glsl::vec2_scalar const&, glsl::vec4_scalar const&, glsl::vec4_scalar const&, NoColor, unsigned int*) /home/twsmith/code/mozilla-central/gfx/wr/swgl/src/swgl_ext.h:701:7
    #3 0x7fc5c6372d61 in brush_image_ALPHA_PASS_ANTIALIASING_REPETITION_TEXTURE_2D_frag::swgl_drawSpanRGBA8() /home/twsmith/code/mozilla-central/objdir-ff-ubsan/x86_64-unknown-linux-gnu/release/build/swgl-17ac762633f478af/out/brush_image_ALPHA_PASS_ANTIALIASING_REPETITION_TEXTURE_2D.h:968:2
    #4 0x7fc5c6366f71 in brush_image_ALPHA_PASS_ANTIALIASING_REPETITION_TEXTURE_2D_frag::draw_span_RGBA8(glsl::FragmentShaderImpl*) /home/twsmith/code/mozilla-central/objdir-ff-ubsan/x86_64-unknown-linux-gnu/release/build/swgl-17ac762633f478af/out/brush_image_ALPHA_PASS_ANTIALIASING_REPETITION_TEXTURE_2D.h:1016:28
    #5 0x7fc5c66c7bd0 in glsl::FragmentShaderImpl::draw_span(unsigned int*, int) /home/twsmith/code/mozilla-central/gfx/wr/swgl/src/program.h:168:12
    #6 0x7fc5c66c7bd0 in void draw_depth_span<unsigned int>(unsigned int, unsigned int*, DepthCursor&) /home/twsmith/code/mozilla-central/gfx/wr/swgl/src/rasterize.h:627:38
    #7 0x7fc5c66c7bd0 in void draw_quad_spans<unsigned int>(int, glsl::vec2_scalar*, unsigned int, glsl::vec3*, Texture&, Texture&, ClipRect const&) /home/twsmith/code/mozilla-central/gfx/wr/swgl/src/rasterize.h:1019:13
    #8 0x7fc5c619da9c in draw_quad(int, Texture&, Texture&) /home/twsmith/code/mozilla-central/gfx/wr/swgl/src/rasterize.h:1615:5
    #9 0x7fc5c619c561 in void draw_elements<unsigned short>(int, int, unsigned long, VertexArray&, Texture&, Texture&) /home/twsmith/code/mozilla-central/gfx/wr/swgl/src/rasterize.h:1645:5
    #10 0x7fc5c619c1ae in DrawElementsInstanced /home/twsmith/code/mozilla-central/gfx/wr/swgl/src/gl.cc:2738:7
    #11 0x7fc5c5abf691 in webrender::device::gl::Device::draw_indexed_triangles_instanced_u16::h2a13a442a4a0fbdf /home/twsmith/code/mozilla-central/gfx/wr/webrender/src/device/gl.rs:3639:9
    #12 0x7fc5c4ef0dc9 in webrender::renderer::Renderer::draw_instanced_batch::h524e1394e2950ff5 /home/twsmith/code/mozilla-central/gfx/wr/webrender/src/renderer/mod.rs:2501:17
    #13 0x7fc5c5d79b77 in webrender::renderer::Renderer::draw_alpha_batch_container::he4b2c7703ec09331 /home/twsmith/code/mozilla-central/gfx/wr/webrender/src/renderer/mod.rs:2994:17
    #14 0x7fc5c5d84546 in webrender::renderer::Renderer::draw_picture_cache_target::hb92d8d40d7fd36b1 /home/twsmith/code/mozilla-central/gfx/wr/webrender/src/renderer/mod.rs:2811:9
    #15 0x7fc5c5d84546 in webrender::renderer::Renderer::draw_frame::hd7890b990cb3c701 /home/twsmith/code/mozilla-central/gfx/wr/webrender/src/renderer/mod.rs:4707:21
    #16 0x7fc5c5d683d8 in webrender::renderer::Renderer::render_impl::hc89b7dbac7001336 /home/twsmith/code/mozilla-central/gfx/wr/webrender/src/renderer/mod.rs:2005:17
    #17 0x7fc5c5d65298 in webrender::renderer::Renderer::render::h050e53d5ddb6b50a /home/twsmith/code/mozilla-central/gfx/wr/webrender/src/renderer/mod.rs:1727:30
    #18 0x7fc5c4d91f1b in wr_renderer_render /home/twsmith/code/mozilla-central/gfx/webrender_bindings/src/bindings.rs:620:11
    #19 0x7fc5b67c9b8e in mozilla::wr::RendererOGL::UpdateAndRender(mozilla::Maybe<mozilla::gfx::IntSizeTyped<mozilla::gfx::UnknownUnits> > const&, mozilla::Maybe<mozilla::wr::ImageFormat> const&, mozilla::Maybe<mozilla::Range<unsigned char> > const&, bool*, mozilla::wr::RendererStats*) /home/twsmith/code/mozilla-central/gfx/webrender_bindings/RendererOGL.cpp:185:8
    #20 0x7fc5b67c8386 in mozilla::wr::RenderThread::UpdateAndRender(mozilla::wr::WrWindowId, mozilla::layers::BaseTransactionId<mozilla::VsyncIdType> const&, mozilla::TimeStamp const&, bool, mozilla::Maybe<mozilla::gfx::IntSizeTyped<mozilla::gfx::UnknownUnits> > const&, mozilla::Maybe<mozilla::wr::ImageFormat> const&, mozilla::Maybe<mozilla::Range<unsigned char> > const&, bool*) /home/twsmith/code/mozilla-central/gfx/webrender_bindings/RenderThread.cpp:533:31
    #21 0x7fc5b67c766b in mozilla::wr::RenderThread::HandleFrameOneDoc(mozilla::wr::WrWindowId, bool) /home/twsmith/code/mozilla-central/gfx/webrender_bindings/RenderThread.cpp:385:3
    #22 0x7fc5b67e8116 in decltype(*(fp).*fp0(Get<0ul>(fp1).PassAsParameter(), Get<1ul>(fp1).PassAsParameter())) mozilla::detail::RunnableMethodArguments<mozilla::wr::WrWindowId, bool>::applyImpl<mozilla::wr::RenderThread, void (mozilla::wr::RenderThread::*)(mozilla::wr::WrWindowId, bool), StoreCopyPassByConstLRef<mozilla::wr::WrWindowId>, StoreCopyPassByConstLRef<bool>, 0ul, 1ul>(mozilla::wr::RenderThread*, void (mozilla::wr::RenderThread::*)(mozilla::wr::WrWindowId, bool), mozilla::Tuple<StoreCopyPassByConstLRef<mozilla::wr::WrWindowId>, StoreCopyPassByConstLRef<bool> >&, std::integer_sequence<unsigned long, 0ul, 1ul>) /home/twsmith/code/mozilla-central/objdir-ff-ubsan/dist/include/nsThreadUtils.h:1147:12
    #23 0x7fc5b67e7edb in decltype(applyImpl(fp, fp0, *(this).mArguments, std::integer_sequence<unsigned long, 0ul, 1ul>{})) mozilla::detail::RunnableMethodArguments<mozilla::wr::WrWindowId, bool>::apply<mozilla::wr::RenderThread, void (mozilla::wr::RenderThread::*)(mozilla::wr::WrWindowId, bool)>(mozilla::wr::RenderThread*, void (mozilla::wr::RenderThread::*)(mozilla::wr::WrWindowId, bool)) /home/twsmith/code/mozilla-central/objdir-ff-ubsan/dist/include/nsThreadUtils.h:1153:12
    #24 0x7fc5b67e7edb in mozilla::detail::RunnableMethodImpl<mozilla::wr::RenderThread*, void (mozilla::wr::RenderThread::*)(mozilla::wr::WrWindowId, bool), true, (mozilla::RunnableKind)0, mozilla::wr::WrWindowId, bool>::Run() /home/twsmith/code/mozilla-central/objdir-ff-ubsan/dist/include/nsThreadUtils.h:1200:13
    #25 0x7fc5b3dd603e in nsThread::ProcessNextEvent(bool, bool*) /home/twsmith/code/mozilla-central/xpcom/threads/nsThread.cpp:1167:16
    #26 0x7fc5b3ddf7e4 in NS_ProcessNextEvent(nsIThread*, bool) /home/twsmith/code/mozilla-central/xpcom/threads/nsThreadUtils.cpp:467:10
    #27 0x7fc5b549f6f4 in mozilla::ipc::MessagePumpForNonMainThreads::Run(base::MessagePump::Delegate*) /home/twsmith/code/mozilla-central/ipc/glue/MessagePump.cpp:330:5
    #28 0x7fc5b530e191 in MessageLoop::RunInternal() /home/twsmith/code/mozilla-central/ipc/chromium/src/base/message_loop.cc:331:10
    #29 0x7fc5b530e191 in MessageLoop::RunHandler() /home/twsmith/code/mozilla-central/ipc/chromium/src/base/message_loop.cc:324:3
    #30 0x7fc5b530e191 in MessageLoop::Run() /home/twsmith/code/mozilla-central/ipc/chromium/src/base/message_loop.cc:306:3
    #31 0x7fc5b3dce7d8 in nsThread::ThreadFunc(void*) /home/twsmith/code/mozilla-central/xpcom/threads/nsThread.cpp:389:10
    #32 0x7fc5de2cb3ee in _pt_root /home/twsmith/code/mozilla-central/nsprpub/pr/src/pthreads/ptthread.c:201:5
    #33 0x7fc5ddef86da in start_thread /build/glibc-uZu3wS/glibc-2.27/nptl/pthread_create.c:463
    #34 0x7fc5dced661e in __clone /build/glibc-uZu3wS/glibc-2.27/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:95

A Pernosco session is available here: https://pernos.co/debug/Ue9RBVJNV8-kLHWhk0FhcQ/index.html

Glenn, is this in your wheelhouse?

Nope, this would be one for Lee.

Division by zero (and also close to zero values) can cause the interpolant step to
become infinite which can feed bogus values into the shader. Since the left and
right edges in this case are essentially at the same position, we can freely just
choose interpolants from either the left or right edge. Just set the step scale
to zero in this case so we default to the interpolant values from the left edge
and don't step them at all in this case.

https://hg.mozilla.org/mozilla-central/rev/6ec5f4ce79c2

The patch landed in nightly and beta is affected.
:lsalzman, is this bug important enough to require an uplift?
If not please set status_beta to wontfix.

For more information, please visit auto_nag documentation.