Fix: multilayered stencil mask issue (#10323)

* tweak stencil modes

* add visual test

* remove only.. again :D

* remove unused vars

---------

Co-authored-by: Zyie <24736175+Zyie@users.noreply.github.com>

src/rendering/mask/stencil/StencilMaskPipe.ts

@@ -150,9 +150,10 @@ export class StencilMaskPipe implements InstructionPipe<StencilMaskInstruction>
             // as no point allocating the memory if we don't use it
             renderer.renderTarget.ensureDepthStencil();
 
+            renderer.stencil.setStencilMode(STENCIL_MODES.RENDERING_MASK_ADD, maskStackIndex);
+
             maskStackIndex++;
 
-            renderer.stencil.setStencilMode(STENCIL_MODES.RENDERING_MASK_ADD, maskStackIndex);
             renderer.colorMask.setMask(0);
         }
         else if (instruction.action === 'pushMaskEnd')
@@ -162,8 +163,6 @@ export class StencilMaskPipe implements InstructionPipe<StencilMaskInstruction>
         }
         else if (instruction.action === 'popMaskBegin')
         {
-            maskStackIndex--;
-
             renderer.colorMask.setMask(0);
 
             if (maskStackIndex !== 0)
@@ -175,6 +174,8 @@ export class StencilMaskPipe implements InstructionPipe<StencilMaskInstruction>
                 renderer.renderTarget.clear(null, CLEAR.STENCIL);
                 renderer.stencil.setStencilMode(STENCIL_MODES.DISABLED, maskStackIndex);
             }
+
+            maskStackIndex--;
         }
         else if (instruction.action === 'popMaskEnd')
         {

src/rendering/renderers/gpu/state/GpuStencilModesToPixi.ts

@@ -25,22 +25,22 @@ GpuStencilModesToPixi[STENCIL_MODES.DISABLED] = {
 
 GpuStencilModesToPixi[STENCIL_MODES.RENDERING_MASK_ADD] = {
     stencilFront: {
-        compare: 'always',
+        compare: 'equal',
         passOp: 'increment-clamp',
     },
     stencilBack: {
-        compare: 'always',
+        compare: 'equal',
         passOp: 'increment-clamp',
     },
 };
 
 GpuStencilModesToPixi[STENCIL_MODES.RENDERING_MASK_REMOVE] = {
     stencilFront: {
-        compare: 'always',
+        compare: 'equal',
         passOp: 'decrement-clamp',
     },
     stencilBack: {
-        compare: 'always',
+        compare: 'equal',
         passOp: 'decrement-clamp',
     },
 };

tests/visual/scenes/mask/layer-mask.scene.ts

@@ -0,0 +1,42 @@
+import { Container } from '../../../../src/scene/container/Container';
+import { Graphics } from '../../../../src/scene/graphics/shared/Graphics';
+
+import type { TestScene } from '../../types';
+
+export const scene: TestScene = {
+    it: 'should render masks if they have multiple layers correctly',
+    create: async (scene: Container) =>
+    {
+        const wrapper = new Container();
+
+        const width = 128 - 20;
+        const height = 128 - 20;
+        const border = 4;
+        const radius = 4;
+
+        const bg = new Graphics()
+            .roundRect(0, 0, width, height, radius + border)
+            .fill(0xffffff)
+            .roundRect(border, border, width - (border * 2), height - (border * 2), radius)
+            .fill(0x0000ff);
+
+        wrapper.x = 10;
+        wrapper.y = 10;
+
+        const mask = new Graphics()
+            .roundRect(0, 0, width, height, radius + border)
+            .fill(0xff0000)
+            .roundRect(border, border, width - (border * 2), height - (border * 2), radius)
+            .fill(0xffffff);
+
+        const thingOutSideofMask = new Graphics()
+            .rect(0, 0, 1000, 20)
+            .fill(0x00ff00);
+
+        //   mask.y =100;
+        wrapper.addChild(mask, bg, thingOutSideofMask);
+        wrapper.mask = mask;
+
+        scene.addChild(wrapper);
+    },
+};