atlas: add support for selection colors

This lets us get rid of SelectionFrom/SelectionTo and the selection
quad.

src/renderer/atlas/AtlasEngine.api.cpp

@@ -7,6 +7,7 @@
 #include "Backend.h"
 #include "../../buffer/out/textBuffer.hpp"
 #include "../base/FontCache.h"
+#include "../../types/inc/ColorFix.hpp"
 
 // #### NOTE ####
 // If you see any code in here that contains "_r." you might be seeing a race condition.
@@ -424,12 +425,14 @@ void AtlasEngine::SetRetroTerminalEffect(bool enable) noexcept
     }
 }
 
-void AtlasEngine::SetSelectionBackground(const COLORREF color, const float alpha) noexcept
+void AtlasEngine::SetSelectionBackground(const COLORREF color, const float /*alpha*/) noexcept
 {
-    const u32 selectionColor = (color & 0xffffff) | gsl::narrow_cast<u32>(lrintf(alpha * 255.0f)) << 24;
+    const u32 selectionColor = (color & 0xffffff) | 0xff000000;
     if (_api.s->misc->selectionColor != selectionColor)
     {
-        _api.s.write()->misc.write()->selectionColor = selectionColor;
+        auto misc = _api.s.write()->misc.write();
+        misc->selectionColor = selectionColor;
+        misc->selectionForeground = 0xff000000 | ColorFix::GetPerceivableColor(color, color, 0.5f * 0.5f);
     }
 }
 

src/renderer/atlas/AtlasEngine.cpp

@@ -311,6 +311,8 @@ CATCH_RETURN()
                 _api.searchHighlightFocused = { info.searchHighlightFocused, 1 };
             }
         }
+
+        _api.selectionSpans = til::point_span_subspan_within_rect(info.selectionSpans, dr);
     }
 
     return S_OK;
@@ -411,9 +413,10 @@ try
         const auto end = isFinalRow ? std::min(hiEnd.x + 1, x2) : x2;
         _fillColorBitmap(row, x1, end, fgColor, bgColor);
 
-        // Return early if we couldn't paint the whole region. We will resume
-        // from here in the next call.
-        if (!isFinalRow || end == x2)
+        // Return early if we couldn't paint the whole region (either this was not the last row, or
+        // it was the last row but the highlight ends outside of our x range.)
+        // We will resume from here in the next call.
+        if (!isFinalRow || hiEnd.x /*inclusive*/ >= x2 /*exclusive*/)
         {
             return S_OK;
         }
@@ -497,6 +500,7 @@ try
     // Apply the highlighting colors to the highlighted cells
     RETURN_IF_FAILED(_drawHighlighted(_api.searchHighlights, y, x, columnEnd, highlightFg, highlightBg));
     RETURN_IF_FAILED(_drawHighlighted(_api.searchHighlightFocused, y, x, columnEnd, highlightFocusFg, highlightFocusBg));
+    RETURN_IF_FAILED(_drawHighlighted(_api.selectionSpans, y, x, columnEnd, _api.s->misc->selectionForeground, _api.s->misc->selectionColor));
 
     _api.lastPaintBufferLineCoord = { x, y };
     return S_OK;
@@ -563,28 +567,9 @@ try
 CATCH_RETURN()
 
 [[nodiscard]] HRESULT AtlasEngine::PaintSelection(const til::rect& rect) noexcept
-try
 {
-    // Unfortunately there's no step after Renderer::_PaintBufferOutput that
-    // would inform us that it's done with the last AtlasEngine::PaintBufferLine.
-    // As such we got to call _flushBufferLine() here just to be sure.
-    _flushBufferLine();
-
-    const auto y = gsl::narrow_cast<u16>(clamp<til::CoordType>(rect.top, 0, _p.s->viewportCellCount.y - 1));
-    const auto from = gsl::narrow_cast<u16>(clamp<til::CoordType>(rect.left, 0, _p.s->viewportCellCount.x - 1));
-    const auto to = gsl::narrow_cast<u16>(clamp<til::CoordType>(rect.right, from, _p.s->viewportCellCount.x));
-
-    auto& row = *_p.rows[y];
-    row.selectionFrom = from;
-    row.selectionTo = to;
-
-    _p.dirtyRectInPx.left = std::min(_p.dirtyRectInPx.left, from * _p.s->font->cellSize.x);
-    _p.dirtyRectInPx.top = std::min(_p.dirtyRectInPx.top, y * _p.s->font->cellSize.y);
-    _p.dirtyRectInPx.right = std::max(_p.dirtyRectInPx.right, to * _p.s->font->cellSize.x);
-    _p.dirtyRectInPx.bottom = std::max(_p.dirtyRectInPx.bottom, _p.dirtyRectInPx.top + _p.s->font->cellSize.y);
     return S_OK;
 }
-CATCH_RETURN()
 
 [[nodiscard]] HRESULT AtlasEngine::PaintCursor(const CursorOptions& options) noexcept
 try

src/renderer/atlas/AtlasEngine.h

@@ -171,6 +171,7 @@ namespace Microsoft::Console::Render::Atlas
             // These tracks the highlighted regions on the screen that are yet to be painted.
             std::span<const til::point_span> searchHighlights;
             std::span<const til::point_span> searchHighlightFocused;
+            std::span<const til::point_span> selectionSpans;
 
             // dirtyRect is a computed value based on invalidatedRows.
             til::rect dirtyRect;

src/renderer/atlas/BackendD2D.cpp

@@ -51,7 +51,6 @@ void BackendD2D::Render(RenderingPayload& p)
         _drawCursorPart1(p);
         _drawText(p);
         _drawCursorPart2(p);
-        _drawSelection(p);
 #if ATLAS_DEBUG_SHOW_DIRTY
         _debugShowDirty(p);
 #endif
@@ -938,26 +937,6 @@ void BackendD2D::_drawCursor(const RenderingPayload& p, ID2D1RenderTarget* rende
     }
 }
 
-void BackendD2D::_drawSelection(const RenderingPayload& p)
-{
-    u16 y = 0;
-    for (const auto& row : p.rows)
-    {
-        if (row->selectionTo > row->selectionFrom)
-        {
-            const D2D1_RECT_F rect{
-                static_cast<f32>(p.s->font->cellSize.x * row->selectionFrom),
-                static_cast<f32>(p.s->font->cellSize.y * y),
-                static_cast<f32>(p.s->font->cellSize.x * row->selectionTo),
-                static_cast<f32>(p.s->font->cellSize.y * (y + 1)),
-            };
-            _fillRectangle(rect, p.s->misc->selectionColor);
-        }
-
-        y++;
-    }
-}
-
 #if ATLAS_DEBUG_SHOW_DIRTY
 void BackendD2D::_debugShowDirty(const RenderingPayload& p)
 {

src/renderer/atlas/BackendD3D.cpp

@@ -229,7 +229,6 @@ void BackendD3D::Render(RenderingPayload& p)
     _drawBackground(p);
     _drawCursorBackground(p);
     _drawText(p);
-    _drawSelection(p);
     _debugShowDirty(p);
     _flushQuads(p);
 
@@ -2248,45 +2247,6 @@ size_t BackendD3D::_drawCursorForegroundSlowPath(const CursorRect& c, size_t off
     return addedInstances;
 }
 
-void BackendD3D::_drawSelection(const RenderingPayload& p)
-{
-    u16 y = 0;
-    u16 lastFrom = 0;
-    u16 lastTo = 0;
-
-    for (const auto& row : p.rows)
-    {
-        if (row->selectionTo > row->selectionFrom)
-        {
-            // If the current selection line matches the previous one, we can just extend the previous quad downwards.
-            // The way this is implemented isn't very smart, but we also don't have very many rows to iterate through.
-            if (row->selectionFrom == lastFrom && row->selectionTo == lastTo)
-            {
-                _getLastQuad().size.y += p.s->font->cellSize.y;
-            }
-            else
-            {
-                _appendQuad() = {
-                    .shadingType = static_cast<u16>(ShadingType::Selection),
-                    .position = {
-                        static_cast<i16>(p.s->font->cellSize.x * row->selectionFrom),
-                        static_cast<i16>(p.s->font->cellSize.y * y),
-                    },
-                    .size = {
-                        static_cast<u16>(p.s->font->cellSize.x * (row->selectionTo - row->selectionFrom)),
-                        static_cast<u16>(p.s->font->cellSize.y),
-                    },
-                    .color = p.s->misc->selectionColor,
-                };
-                lastFrom = row->selectionFrom;
-                lastTo = row->selectionTo;
-            }
-        }
-
-        y++;
-    }
-}
-
 void BackendD3D::_debugShowDirty(const RenderingPayload& p)
 {
 #if ATLAS_DEBUG_SHOW_DIRTY
@@ -2302,7 +2262,7 @@ void BackendD3D::_debugShowDirty(const RenderingPayload& p)
     {
         const auto& rect = _presentRects[(_presentRectsPos + i) % std::size(_presentRects)];
         _appendQuad() = {
-            .shadingType = static_cast<u16>(ShadingType::Selection),
+            .shadingType = static_cast<u16>(ShadingType::FilledRect),
             .position = {
                 static_cast<i16>(rect.left),
                 static_cast<i16>(rect.top),

src/renderer/atlas/BackendD3D.h

@@ -76,7 +76,7 @@ namespace Microsoft::Console::Render::Atlas
             SolidLine,
 
             Cursor,
-            Selection,
+            FilledRect,
 
             TextDrawingFirst = TextGrayscale,
             TextDrawingLast = SolidLine,

src/renderer/atlas/common.h

@@ -393,7 +393,8 @@ namespace Microsoft::Console::Render::Atlas
     struct MiscellaneousSettings
     {
         u32 backgroundColor = 0;
-        u32 selectionColor = 0x7fffffff;
+        u32 selectionColor = 0xffffffff;
+        u32 selectionForeground = 0xff000000;
         std::wstring customPixelShaderPath;
         std::wstring customPixelShaderImagePath;
         bool useRetroTerminalEffect = false;
@@ -475,8 +476,6 @@ namespace Microsoft::Console::Render::Atlas
             bitmap.active = false;
             gridLineRanges.clear();
             lineRendition = LineRendition::SingleWidth;
-            selectionFrom = 0;
-            selectionTo = 0;
             dirtyTop = y * cellHeight;
             dirtyBottom = dirtyTop + cellHeight;
         }
@@ -496,8 +495,6 @@ namespace Microsoft::Console::Render::Atlas
         Bitmap bitmap;
         std::vector<GridLineRange> gridLineRanges;
         LineRendition lineRendition = LineRendition::SingleWidth;
-        u16 selectionFrom = 0;
-        u16 selectionTo = 0;
         til::CoordType dirtyTop = 0;
         til::CoordType dirtyBottom = 0;
     };

src/renderer/atlas/shader_common.hlsl

@@ -16,7 +16,7 @@
 #define SHADING_TYPE_CURLY_LINE         7
 #define SHADING_TYPE_SOLID_LINE         8
 #define SHADING_TYPE_CURSOR             9
-#define SHADING_TYPE_SELECTION          10
+#define SHADING_TYPE_FILLED_RECT       10
 
 struct VSData
 {