Refactor mouse freezing and fix some minor bugs.

VGLMouseFreeze() now only defers mouse signals and leaves it to higher
levels to hide and unhide the mouse cursor if necessary.  (It is never
necessary, but is done to simplify the implementation.  It is slow and
flashes the cursor.  It is still done for copying bitmaps and clearing.)

VGLMouseUnFreeze() now only undoes 1 level of freezing.  Its old
optimization to reduce mouse redrawing is too hard to do with unhiding
in higher levels, and its undoing of multiple levels was a historical
mistake.

VGLMouseOverlap() determines if a region overlaps the (full) mouse region.

VGLMouseFreezeXY() is the freezing and a precise overlap check combined
for the special case of writing a single pixel.  This is the single-pixel
case of the old VGLMouseFreeze() with cleanups.

Fixes:
- check in more cases that the application didn't pass an invalid VIDBUF
- check for errors from copying a bitmap to the shadow buffer
- freeze the mouse before writing to the shadow buffer in all cases.  This
  was not done for the case of writing a single pixel (there was a race)
- don't spell the #defined values for VGLMouseShown as 0, 1 or boolean.

lib/libvgl/bitmap.c

@@ -214,23 +214,36 @@ int
 VGLBitmapCopy(VGLBitmap *src, int srcx, int srcy,
 	      VGLBitmap *dst, int dstx, int dsty, int width, int hight)
 {
-  int error;
+  int error, mouseoverlap;
 
   if (src == VGLDisplay)
     src = &VGLVDisplay;
   if (src->Type != MEMBUF)
     return -1;		/* invalid */
   if (dst == VGLDisplay) {
-    VGLMouseFreeze(dstx, dsty, width, hight, 0);
-    __VGLBitmapCopy(src, srcx, srcy, &VGLVDisplay, dstx, dsty, width, hight);
+    VGLMouseFreeze();
+    mouseoverlap = VGLMouseOverlap(dstx, dsty, width, hight);
+    if (mouseoverlap)
+      VGLMousePointerHide();
+    error = __VGLBitmapCopy(src, srcx, srcy, &VGLVDisplay, dstx, dsty,
+                            width, hight);
+    if (error != 0) {
+      if (mouseoverlap)
+        VGLMousePointerShow();
+      VGLMouseUnFreeze();
+      return error;
+    }
     src = &VGLVDisplay;
     srcx = dstx;
     srcy = dsty;
   } else if (dst->Type != MEMBUF)
     return -1;		/* invalid */
   error = __VGLBitmapCopy(src, srcx, srcy, dst, dstx, dsty, width, hight);
-  if (dst == VGLDisplay)
+  if (dst == VGLDisplay) {
+    if (mouseoverlap)
+      VGLMousePointerShow();
     VGLMouseUnFreeze();
+  }
   return error;
 }
 

lib/libvgl/mouse.c

@@ -89,7 +89,7 @@ static VGLBitmap VGLMouseStdOrMask =
     VGLBITMAP_INITIALIZER(MEMBUF, MOUSE_IMG_SIZE, MOUSE_IMG_SIZE, StdOrMask);
 static VGLBitmap *VGLMouseAndMask, *VGLMouseOrMask;
 static int VGLMouseVisible = 0;
-static int VGLMouseShown = 0;
+static int VGLMouseShown = VGL_MOUSEHIDE;
 static int VGLMouseXpos = 0;
 static int VGLMouseYpos = 0;
 static int VGLMouseButtons = 0;
@@ -316,48 +316,47 @@ VGLMouseStatus(int *x, int *y, char *buttons)
   return VGLMouseShown;
 }
 
-int
-VGLMouseFreeze(int x, int y, int width, int hight, u_long color)
+void
+VGLMouseFreeze(void)
 {
-    INTOFF();
-    if (width > 1 || hight > 1 || (color & 0xc0000000) == 0) { /* bitmap */
-      if (VGLMouseShown == 1) {
-        int overlap;
+  INTOFF();
+}
 
-        if (x > VGLMouseXpos)
-          overlap = (VGLMouseXpos + MOUSE_IMG_SIZE) - x;
-        else
-          overlap = (x + width) - VGLMouseXpos;
-        if (overlap > 0) {
-          if (y > VGLMouseYpos)
-            overlap = (VGLMouseYpos + MOUSE_IMG_SIZE) - y;
-          else
-            overlap = (y + hight) - VGLMouseYpos;
-          if (overlap > 0)
-            VGLMousePointerHide();
-        } 
-      }
-    }
-    else {				/* bit */
-      if (VGLMouseShown &&
-          x >= VGLMouseXpos && x < VGLMouseXpos + MOUSE_IMG_SIZE &&
-          y >= VGLMouseYpos && y < VGLMouseYpos + MOUSE_IMG_SIZE) {
-        if (color & 0x80000000) {	/* Set */
-          if (VGLMouseAndMask->Bitmap 
-            [(y-VGLMouseYpos)*MOUSE_IMG_SIZE+(x-VGLMouseXpos)]) {
-            return 1;
-          }   
-        }   
-      }       
-    }
+int
+VGLMouseFreezeXY(int x, int y)
+{
+  INTOFF();
+  if (VGLMouseShown != VGL_MOUSESHOW)
+    return 0;
+  if (x >= VGLMouseXpos && x < VGLMouseXpos + MOUSE_IMG_SIZE &&
+      y >= VGLMouseYpos && y < VGLMouseYpos + MOUSE_IMG_SIZE &&
+      VGLMouseAndMask->Bitmap[(y-VGLMouseYpos)*MOUSE_IMG_SIZE+(x-VGLMouseXpos)])
+    return 1;
   return 0;
 }
 
+int
+VGLMouseOverlap(int x, int y, int width, int hight)
+{
+  int overlap;
+
+  if (VGLMouseShown != VGL_MOUSESHOW)
+    return 0;
+  if (x > VGLMouseXpos)
+    overlap = (VGLMouseXpos + MOUSE_IMG_SIZE) - x;
+  else
+    overlap = (x + width) - VGLMouseXpos;
+  if (overlap <= 0)
+    return 0;
+  if (y > VGLMouseYpos)
+    overlap = (VGLMouseYpos + MOUSE_IMG_SIZE) - y;
+  else
+    overlap = (y + hight) - VGLMouseYpos;
+  return overlap > 0;
+}
+
 void
 VGLMouseUnFreeze()
 {
-  if (VGLMouseShown == VGL_MOUSESHOW && !VGLMouseVisible && !VGLMintpending)
-    VGLMousePointerShow();
-  while (VGLMsuppressint)
-    INTON();
+  INTON();
 }

lib/libvgl/simple.c

@@ -51,14 +51,18 @@ static byte VGLSavePaletteBlue[256];
 void
 VGLSetXY(VGLBitmap *object, int x, int y, u_long color)
 {
-  int offset;
+  int offset, undermouse;
 
   VGLCheckSwitch();
   if (x>=0 && x<object->VXsize && y>=0 && y<object->VYsize) {
-    if (object == VGLDisplay)
+    if (object == VGLDisplay) {
+      undermouse = VGLMouseFreezeXY(x, y);
       VGLSetXY(&VGLVDisplay, x, y, color);
-    if (object->Type == MEMBUF ||
-        !VGLMouseFreeze(x, y, 1, 1, 0x80000000 | color)) {
+    } else if (object->Type != MEMBUF)
+      return;		/* invalid */
+    else
+      undermouse = 0;
+    if (!undermouse) {
       offset = (y * object->VXsize + x) * object->PixelBytes;
       switch (object->Type) {
       case VIDBUF8S:
@@ -106,7 +110,7 @@ VGLSetXY(VGLBitmap *object, int x, int y, u_long color)
 	object->Bitmap[offset] |= (byte)color;
       }
     }
-    if (object->Type != MEMBUF)
+    if (object == VGLDisplay)
       VGLMouseUnFreeze();
   }
 }
@@ -143,7 +147,7 @@ VGLGetXY(VGLBitmap *object, int x, int y)
     return 0;
   if (object == VGLDisplay)
     object = &VGLVDisplay;
-  if (object->Type != MEMBUF)
+  else if (object->Type != MEMBUF)
     return 0;		/* invalid */
   return __VGLGetXY(object, x, y);
 }
@@ -449,13 +453,14 @@ void
 VGLClear(VGLBitmap *object, u_long color)
 {
   VGLBitmap src;
-  int offset;
-  int len;
-  int i;
+  int i, len, mouseoverlap, offset;
 
   VGLCheckSwitch();
   if (object == VGLDisplay) {
-    VGLMouseFreeze(0, 0, object->Xsize, object->Ysize, color);
+    VGLMouseFreeze();
+    mouseoverlap = VGLMouseOverlap(0, 0, object->Xsize, object->Ysize);
+    if (mouseoverlap)
+      VGLMousePointerHide();
     VGLClear(&VGLVDisplay, color);
   } else if (object->Type != MEMBUF)
     return;		/* invalid */
@@ -509,8 +514,11 @@ VGLClear(VGLBitmap *object, u_long color)
     outb(0x3ce, 0x05); outb(0x3cf, 0x00);
     break;
   }
-  if (object == VGLDisplay)
+  if (object == VGLDisplay) {
+    if (mouseoverlap)
+      VGLMousePointerShow();
     VGLMouseUnFreeze();
+  }
 }
 
 static inline u_long

lib/libvgl/vgl.h

@@ -134,7 +134,9 @@ void VGLMouseSetStdImage(void);
 int VGLMouseInit(int mode);
 void VGLMouseRestore(void);
 int VGLMouseStatus(int *x, int *y, char *buttons);
-int VGLMouseFreeze(int x, int y, int width, int hight, u_long color);
+void VGLMouseFreeze(void);
+int VGLMouseFreezeXY(int x, int y);
+int VGLMouseOverlap(int x, int y, int width, int hight);
 void VGLMouseUnFreeze(void);
 /* simple.c */
 void VGLSetXY(VGLBitmap *object, int x, int y, u_long color);