Feature: #27: Detect shaped windows and disable shadow on them

- Optionally detect shaped windows using X Shape extension and disable
  shadow on them with --shadow-ignore-shaped.

- Some windows are bounding-shaped just to support rounded corners, like
  Chromium windows (when system titlebar is disabled in its settings).
  Add --detect-rounded-corners to treat them as non-shaped windows (thus
  enable shadow on them). The algorithm I use is not perfect and wrong
  detection results are pretty possible to appear.

- Many windows don't use X Shape extensions to add shapes but use ARGB
  background instead. These windows could only be blacklisted with
  --shadow-blacklist.

- Rename a few functions. Code clean up.

1 files changed, 82 insertions, 46 deletions

diff --git a/compton.h b/compton.h
index 3fd4541a4..83b78769e 100644
--- a/compton.h
+++ b/compton.h
@@ -75,6 +75,9 @@
 #define HAS_NAME_WINDOW_PIXMAP 1
 #endif
 
+#define ROUNDED_PERCENT 0.05
+#define ROUNDED_PIXELS  10
+
 // For printing timestamps
 #include <time.h>
 extern struct timeval time_start;
@@ -174,6 +177,10 @@ typedef struct _win {
   Bool destroyed;
   /// Cached width/height of the window including border.
   int widthb, heightb;
+  /// Whether the window is bounding-shaped.
+  Bool bounding_shaped;
+  /// Whether the window just have rounded corners.
+  Bool rounded_corners;
 
   // Blacklist related members
   char *name;
@@ -257,6 +264,9 @@ typedef struct _options {
   Bool mark_ovredir_focused;
   /// Whether to fork to background.
   Bool fork_after_register;
+  /// Whether to detect rounded corners.
+  Bool detect_rounded_corners;
+  /// Whether to work under synchronized mode for debugging.
   Bool synchronize;
 
   // Shadow
@@ -269,6 +279,8 @@ typedef struct _options {
   Bool clear_shadow;
   /// Shadow blacklist. A linked list of conditions.
   wincond *shadow_blacklist;
+  /// Whether bounding-shaped window should be ignored.
+  Bool shadow_ignore_shaped;
 
   // Fading
   Bool wintype_fade[NUM_WINTYPES];
@@ -322,6 +334,7 @@ typedef enum {
 extern int root_height, root_width;
 extern Atom atom_client_attr;
 extern Bool idling;
+extern Bool shape_exists;
 
 /**
  * Functions
@@ -563,52 +576,6 @@ free_damage(Display *dpy, Damage *p) {
   }
 }
 
-/**
- * Determine if a window has a specific attribute.
- *
- * @param dpy Display to use
- * @param w window to check
- * @param atom atom of attribute to check
- * @return 1 if it has the attribute, 0 otherwise
- */
-static inline Bool
-win_has_attr(Display *dpy, Window w, Atom atom) {
-  Atom type = None;
-  int format;
-  unsigned long nitems, after;
-  unsigned char *data;
-
-  if (Success == XGetWindowProperty(dpy, w, atom, 0, 0, False,
-        AnyPropertyType, &type, &format, &nitems, &after, &data)) {
-    XFree(data);
-    if (type) return True;
-  }
-
-  return False;
-}
-
-/**
- * Get the children of a window.
- *
- * @param dpy Display to use
- * @param w window to check
- * @param children [out] an array of child window IDs
- * @param nchildren [out] number of children
- * @return 1 if successful, 0 otherwise
- */
-static inline Bool
-win_get_children(Display *dpy, Window w,
-    Window **children, unsigned *nchildren) {
-  Window troot, tparent;
-
-  if (!XQueryTree(dpy, w, &troot, &tparent, children, nchildren)) {
-    *nchildren = 0;
-    return False;
-  }
-
-  return True;
-}
-
 static unsigned long
 get_time_in_milliseconds(void);
 
@@ -668,6 +635,75 @@ static inline bool is_normal_win(const win *w) {
       || WINTYPE_UNKNOWN == w->window_type);
 }
 
+/**
+ * Determine if a window has a specific attribute.
+ *
+ * @param dpy Display to use
+ * @param w window to check
+ * @param atom atom of attribute to check
+ * @return 1 if it has the attribute, 0 otherwise
+ */
+static inline Bool
+wid_has_attr(Display *dpy, Window w, Atom atom) {
+  Atom type = None;
+  int format;
+  unsigned long nitems, after;
+  unsigned char *data;
+
+  if (Success == XGetWindowProperty(dpy, w, atom, 0, 0, False,
+        AnyPropertyType, &type, &format, &nitems, &after, &data)) {
+    XFree(data);
+    if (type) return True;
+  }
+
+  return False;
+}
+
+/**
+ * Get the children of a window.
+ *
+ * @param dpy Display to use
+ * @param w window to check
+ * @param children [out] an array of child window IDs
+ * @param nchildren [out] number of children
+ * @return 1 if successful, 0 otherwise
+ */
+static inline Bool
+wid_get_children(Display *dpy, Window w,
+    Window **children, unsigned *nchildren) {
+  Window troot, tparent;
+
+  if (!XQueryTree(dpy, w, &troot, &tparent, children, nchildren)) {
+    *nchildren = 0;
+    return False;
+  }
+
+  return True;
+}
+
+/**
+ * Check if a window is bounding-shaped.
+ */
+static inline Bool
+wid_bounding_shaped(Display *dpy, Window wid) {
+  if (shape_exists) {
+    Bool bounding_shaped = False;
+    Bool clip_shaped;
+    int x_bounding, y_bounding, x_clip, y_clip;
+    unsigned int w_bounding, h_bounding, w_clip, h_clip;
+
+    XShapeQueryExtents(dpy, wid, &bounding_shaped,
+        &x_bounding, &y_bounding, &w_bounding, &h_bounding,
+        &clip_shaped, &x_clip, &y_clip, &w_clip, &h_clip);
+    return bounding_shaped;
+  }
+  
+  return False;
+}
+
+static void
+win_rounded_corners(Display *dpy, win *w);
+
 static bool
 win_match_once(win *w, const wincond *cond);