Bug fix: GLX backend incompatibility with VirtualBox & others

- GLX backend: Fix a bug that window content does not get updated on
  VirtualBox, by rebinding texture when window content changes. This may
  have a negative effect on performance.

- GLX backend: Add --glx-no-stencil to restore the old clipping method,
  just in case.

- GLX backend: Apply stricter checks on texture-pixmap binding.

- GLX backend: Fix a bug that glx_set_clip() behaves incorrectly when
  None is passed in.

- GLX backend: Use glEnable()/glDisable() to toggle stencil tests, in
  hope to increase performance.

- Move window pixmap/picture fetching to win_paint_win(), in hope to
  increase performance.

- Intersect shadow painting region with its bounding rectangle, in hope
  to increase performance.

4 files changed, 89 insertions, 46 deletions

diff --git a/common.h b/common.h
index f36edf4ba..77aa582f6 100644
--- a/common.h
+++ b/common.h
@@ -339,6 +339,8 @@ typedef struct {
   char *display;
   /// The backend in use.
   enum backend backend;
+  /// Whether to avoid using stencil buffer under GLX backend. Might be unsafe.
+  bool glx_no_stencil;
   /// Whether to try to detect WM windows and mark them as focused.
   bool mark_wmwin_focused;
   /// Whether to mark override-redirect windows as focused.
@@ -724,6 +726,8 @@ typedef struct _win {
   winmode_t mode;
   /// Whether the window has been damaged at least once.
   bool damaged;
+  /// Whether the window was damaged after last paint.
+  bool pixmap_damaged;
   /// Damage of the window.
   Damage damage;
   /// Paint info of the window.
@@ -1574,9 +1578,13 @@ glx_bind_pixmap(session_t *ps, glx_texture_t **pptex, Pixmap pixmap,
 void
 glx_release_pixmap(session_t *ps, glx_texture_t *ptex);
 
+/**
+ * Check if a texture is binded, or is binded to the given pixmap.
+ */
 static inline bool
-glx_tex_binded(const glx_texture_t *ptex) {
-  return ptex && ptex->glpixmap && ptex->texture;
+glx_tex_binded(const glx_texture_t *ptex, Pixmap pixmap) {
+  return ptex && ptex->glpixmap && ptex->texture
+    && (!pixmap || pixmap == ptex->pixmap);
 }
 
 void
diff --git a/compton.c b/compton.c
index df39a36f4..591fc7043 100644
--- a/compton.c
+++ b/compton.c
@@ -474,7 +474,7 @@ win_build_shadow(session_t *ps, win *w, double opacity) {
 
   w->shadow_paint.pixmap = shadow_pixmap_argb;
   w->shadow_paint.pict = shadow_picture_argb;
-  bool success = paint_bind_tex(ps, &w->shadow_paint, shadow_image->width, shadow_image->height, 32);
+  bool success = paint_bind_tex(ps, &w->shadow_paint, shadow_image->width, shadow_image->height, 32, true);
 
   XFreeGC(ps->dpy, gc);
   XDestroyImage(shadow_image);
@@ -1281,36 +1281,6 @@ paint_preprocess(session_t *ps, win *list) {
     redir_start(ps);
   }
 
-  // Fetch pictures only if windows are redirected
-  if (ps->redirected) {
-    for (w = t; w; w = w->prev_trans) {
-      // Fetch Pixmap
-      if (!w->paint.pixmap && ps->has_name_pixmap) {
-          set_ignore_next(ps);
-          w->paint.pixmap = XCompositeNameWindowPixmap(ps->dpy, w->id);
-      }
-      // XRender: Build picture
-      if (BKEND_XRENDER == ps->o.backend && !w->paint.pict) {
-        Drawable draw = w->paint.pixmap;
-        if (!draw)
-          draw = w->id;
-        {
-          XRenderPictureAttributes pa = {
-            .subwindow_mode = IncludeInferiors,
-          };
-
-          w->paint.pict = XRenderCreatePicture(ps->dpy, draw, w->pictfmt,
-              CPSubwindowMode, &pa);
-        }
-      }
-      // OpenGL: Build texture
-      if (!paint_bind_tex(ps, &w->paint, w->widthb, w->heightb,
-            w->pictfmt->depth)) {
-        printf_errf("(%#010lx): Failed to bind texture. Expect troubles.", w->id);
-      }
-    }
-  }
-
   return t;
 }
 
@@ -1447,6 +1417,32 @@ render(session_t *ps, int x, int y, int dx, int dy, int wid, int hei,
  */
 static inline void
 win_paint_win(session_t *ps, win *w, XserverRegion reg_paint) {
+  // Fetch Pixmap
+  if (!w->paint.pixmap && ps->has_name_pixmap) {
+    set_ignore_next(ps);
+    w->paint.pixmap = XCompositeNameWindowPixmap(ps->dpy, w->id);
+  }
+  // XRender: Build picture
+  if (BKEND_XRENDER == ps->o.backend && !w->paint.pict) {
+    Drawable draw = w->paint.pixmap;
+    if (!draw)
+      draw = w->id;
+    {
+      XRenderPictureAttributes pa = {
+        .subwindow_mode = IncludeInferiors,
+      };
+
+      w->paint.pict = XRenderCreatePicture(ps->dpy, draw, w->pictfmt,
+          CPSubwindowMode, &pa);
+    }
+  }
+  // GLX: Build texture
+  if (!paint_bind_tex(ps, &w->paint, w->widthb, w->heightb,
+        w->pictfmt->depth, w->pixmap_damaged)) {
+    printf_errf("(%#010lx): Failed to bind texture. Expect troubles.", w->id);
+  }
+  w->pixmap_damaged = false;
+
   if (!paint_isvalid(ps, &w->paint)) {
     printf_errf("(%#010lx): Missing painting data. This is a bad sign.", w->id);
     return;
@@ -1664,6 +1660,21 @@ paint_all(session_t *ps, XserverRegion region, win *t) {
         reg_paint = reg_tmp;
         XFixesIntersectRegion(ps->dpy, reg_paint, region, w->extents);
       }
+
+      // Might be worthwhile to crop the region to shadow border
+      {
+        XRectangle rec_shadow_border = {
+          .x = w->a.x + w->shadow_dx,
+          .y = w->a.y + w->shadow_dy,
+          .width = w->shadow_width,
+          .height = w->shadow_height
+        };
+        XserverRegion reg_shadow = XFixesCreateRegion(ps->dpy,
+            &rec_shadow_border, 1);
+        XFixesIntersectRegion(ps->dpy, reg_paint, reg_paint, reg_shadow);
+        free_region(ps, &reg_shadow);
+      }
+
       // Clear the shadow here instead of in make_shadow() for saving GPU
       // power and handling shaped windows
       if (ps->o.clear_shadow && w->border_size)
@@ -1837,6 +1848,7 @@ repair_win(session_t *ps, win *w) {
 
   add_damage(ps, parts);
   w->damaged = true;
+  w->pixmap_damaged = true;
 }
 
 static wintype_t
@@ -2469,6 +2481,7 @@ add_win(session_t *ps, Window id, Window prev) {
     .mode = WMODE_TRANS,
     .damaged = false,
     .damage = None,
+    .pixmap_damaged = false,
     .paint = PAINT_INIT,
     .border_size = None,
     .extents = None,
@@ -4104,6 +4117,10 @@ usage(void) {
     "  inverted color. Resource-hogging, and is not well tested.\n"
     "--backend backend\n"
     "  Choose backend. Possible choices are xrender and glx" WARNING ".\n"
+    "--glx-no-stencil\n"
+    "  Avoid using stencil buffer under GLX backend. Might cause issues\n"
+    "  when rendering transparent content, may have a positive or\n"
+    "  negative effect on performance.\n"
 #undef WARNING
 #ifndef CONFIG_DBUS
 #define WARNING WARNING_DISABLED
@@ -4560,6 +4577,7 @@ get_cfg(session_t *ps, int argc, char *const *argv, bool first_pass) {
     { "invert-color-include", required_argument, NULL, 288 },
     { "opengl", no_argument, NULL, 289 },
     { "backend", required_argument, NULL, 290 },
+    { "glx-no-stencil", no_argument, NULL, 291 },
     // Must terminate with a NULL entry
     { NULL, 0, NULL, 0 },
   };
@@ -4822,6 +4840,10 @@ get_cfg(session_t *ps, int argc, char *const *argv, bool first_pass) {
         if (!parse_backend(ps, optarg))
           exit(1);
         break;
+      case 291:
+        // --glx-no-stencil
+        ps->o.glx_no_stencil = true;
+        break;
       default:
         usage();
         break;
@@ -5619,6 +5641,7 @@ session_init(session_t *ps_old, int argc, char **argv) {
       .config_file = NULL,
       .display = NULL,
       .backend = BKEND_XRENDER,
+      .glx_no_stencil = false,
       .mark_wmwin_focused = false,
       .mark_ovredir_focused = false,
       .fork_after_register = false,
diff --git a/compton.h b/compton.h
index 49ffcb88f..9c4bd0a9f 100644
--- a/compton.h
+++ b/compton.h
@@ -169,7 +169,7 @@ paint_isvalid(session_t *ps, const paint_t *ppaint) {
     return false;
 
 #ifdef CONFIG_VSYNC_OPENGL
-  if (BKEND_GLX == ps->o.backend && !glx_tex_binded(ppaint->ptex))
+  if (BKEND_GLX == ps->o.backend && !glx_tex_binded(ppaint->ptex, None))
     return false;
 #endif
 
@@ -179,10 +179,12 @@ paint_isvalid(session_t *ps, const paint_t *ppaint) {
  * Bind texture in paint_t if we are using GLX backend.
  */
 static inline bool
-paint_bind_tex(session_t *ps, paint_t *ppaint, int wid, int hei, int depth) {
+paint_bind_tex(session_t *ps, paint_t *ppaint, int wid, int hei, int depth,
+    bool force) {
 #ifdef CONFIG_VSYNC_OPENGL
   // TODO: Make sure we have the same Pixmap binded?
-  if (BKEND_GLX == ps->o.backend && !glx_tex_binded(ppaint->ptex)) {
+  if (BKEND_GLX == ps->o.backend
+      && (force || !glx_tex_binded(ppaint->ptex, ppaint->pixmap))) {
     return glx_bind_pixmap(ps, &ppaint->ptex, ppaint->pixmap, wid, hei, depth);
   }
 #endif
diff --git a/opengl.c b/opengl.c
index 2ec0b3313..b32368e2f 100644
--- a/opengl.c
+++ b/opengl.c
@@ -71,7 +71,7 @@ glx_init(session_t *ps, bool need_render) {
   // Ensure we have a stencil buffer. X Fixes does not guarantee rectangles
   // in regions don't overlap, so we must use stencil buffer to make sure
   // we don't paint a region for more than one time, I think?
-  if (need_render) {
+  if (need_render && !ps->o.glx_no_stencil) {
     GLint val = 0;
     glGetIntegerv(GL_STENCIL_BITS, &val);
     if (!val) {
@@ -110,11 +110,13 @@ glx_init(session_t *ps, bool need_render) {
     glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
     glDisable(GL_BLEND);
 
-    // Initialize stencil buffer
-    glClear(GL_STENCIL_BUFFER_BIT);
-    glEnable(GL_STENCIL_TEST);
-    glStencilMask(0x1);
-    glStencilFunc(GL_EQUAL, 0x1, 0x1);
+    if (!ps->o.glx_no_stencil) {
+      // Initialize stencil buffer
+      glClear(GL_STENCIL_BUFFER_BIT);
+      glDisable(GL_STENCIL_TEST);
+      glStencilMask(0x1);
+      glStencilFunc(GL_EQUAL, 0x1, 0x1);
+    }
 
     // Clear screen
     glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
@@ -466,9 +468,14 @@ glx_release_pixmap(session_t *ps, glx_texture_t *ptex) {
  */
 void
 glx_set_clip(session_t *ps, XserverRegion reg) {
-  glClear(GL_STENCIL_BUFFER_BIT);
+  // Quit if we aren't using stencils
+  if (ps->o.glx_no_stencil)
+    return;
 
   if (reg) {
+    glEnable(GL_STENCIL_TEST);
+    glClear(GL_STENCIL_BUFFER_BIT);
+
     glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
     glDepthMask(GL_FALSE);
     glStencilOp(GL_REPLACE, GL_KEEP, GL_KEEP);
@@ -504,6 +511,9 @@ glx_set_clip(session_t *ps, XserverRegion reg) {
     glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
     glDepthMask(GL_TRUE);
   }
+  else {
+    glDisable(GL_STENCIL_TEST);
+  }
 }
 
 /**
@@ -560,15 +570,15 @@ glx_render(session_t *ps, const glx_texture_t *ptex,
     printf_dbgf("(): Draw: %d, %d, %d, %d -> %d, %d (%d, %d) z %d\n", x, y, width, height, dx, dy, ptex->width, ptex->height, z);
 #endif
 
-    /*
-    if (reg_tgt) {
+    // On no-stencil mode, calculate painting region here instead of relying
+    // on stencil buffer
+    if (ps->o.glx_no_stencil && reg_tgt) {
       reg_new = XFixesCreateRegion(ps->dpy, &rec_all, 1);
       XFixesIntersectRegion(ps->dpy, reg_new, reg_new, reg_tgt);
 
       nrects = 0;
       rects = XFixesFetchRegion(ps->dpy, reg_new, &nrects);
     }
-    */
 
     glEnable(GL_TEXTURE_2D);
     glBindTexture(GL_TEXTURE_2D, ptex->texture);