After fiddling a lot with the fragment code I came up with this solution .
gui_maskable_1.fp
varying highp vec2 var_texcoord0;
varying lowp vec4 var_color;
uniform lowp sampler2D texture_sampler;
uniform lowp sampler2D mask_sampler_1;
uniform lowp vec4 texture_position_and_scale_1;
uniform lowp vec4 mask_position_and_scale_1;
uniform lowp vec4 mask_size;
void main()
{
// Sample textures
vec4 tex = texture2D(texture_sampler, (var_texcoord0.xy - texture_position_and_scale_1.xy) / texture_position_and_scale_1.zw);
vec4 masky = texture2D(mask_sampler_1, (var_texcoord0.xy - mask_position_and_scale_1.xy) / mask_position_and_scale_1.zw);
// Calculate alpha multiplier
float alpha_mul = 1.0 - var_color.w * (1.0 - masky.a);
// Check if fragment is within mask bounds
bool isWithinMaskBounds = all(greaterThanEqual(var_texcoord0.xy, mask_position_and_scale_1.xy)) &&
all(lessThanEqual(var_texcoord0.xy, mask_position_and_scale_1.xy + mask_size.xy));
// Discard fragment if outside mask bounds or mask alpha is too low
if (!isWithinMaskBounds || masky.a < 0.1) { // Assuming 0.1 as the threshold for visible alpha
discard;
}
// Apply color and alpha multiplier
tex *= (alpha_mul * masky.r * var_color.w);
gl_FragColor = tex * var_color;
}
and in the render script you must pass the size in normalized form
camera_gui.frustum.constants["mask_size"] = vmath.vector4(gui_maskable.mask_width / gui_maskable.texture_width, gui_maskable.mask_height / gui_maskable.texture_height, 0, 0)