View RSS Feed

Fubarable

Windows Screen Grabber JNA methods that I'm working on

Rate this Entry
by , 05-26-2012 at 05:50 AM (4177 Views)
This of course requires JNA jar files, platform.jar and jna.jar. The ultimate goal of this code is to create a "bot" that can control a work program. Since the work program uses a Citrix client to interact with the user, if the controller program resides on the client it is impossible to get the program state through the usual methods, and instead I will be required to check for changes in the program display to obtain this information. This code is far from done, but does show some ways to interact with the OS via JNA. Comments are sorely lacking, I'm afraid.

jna.User32.java
Java Code:
package jna;

import com.sun.jna.Native;
import com.sun.jna.Pointer;
import com.sun.jna.platform.win32.WinDef.RECT;
import com.sun.jna.win32.StdCallLibrary;

/**
 * JNA interface with Window's user32.dll
 * 
 * @author Pete S
 *
 */
public interface User32 extends StdCallLibrary {
   User32 INSTANCE = (User32) Native.loadLibrary("user32", User32.class);

   interface WNDENUMPROC extends StdCallCallback {
      boolean callback(Pointer hWnd, Pointer arg);
   }
   
   public static final int GW_OWNER = 4; // used with GetWindow to get win owner

   boolean EnumWindows(WNDENUMPROC lpEnumFunc, Pointer userData);
   int GetWindowTextA(Pointer hWnd, byte[] lpString, int nMaxCount);
   int SetForegroundWindow(Pointer hWnd);
   Pointer GetForegroundWindow();
   boolean GetWindowRect(Pointer hWnd, RECT rect);
   boolean IsWindow(Pointer hWnd);
   Pointer GetWindow(Pointer hWnd, int uCmd);
}
jna.JnaUtil.java
Java Code:
package jna;

import java.awt.AWTException;
import java.awt.Rectangle;
import java.awt.Robot;
import java.awt.image.BufferedImage;

import javax.swing.ImageIcon;
import javax.swing.JLabel;
import javax.swing.JOptionPane;

import com.sun.jna.Native;
import com.sun.jna.Pointer;
import com.sun.jna.platform.win32.WinDef.RECT;

/**
 * static methods to allow Java to call Windows code. 
 * user32.dll code is as specified in the JNA interface
 * User32.java
 *  
 * @author Pete S
 *
 */
public class JnaUtil {
   private static final User32 user32 = User32.INSTANCE;
   private static Pointer callBackHwnd;

   public static boolean windowExists(final String startOfWindowName) {
      return !user32.EnumWindows(new User32.WNDENUMPROC() {
         @Override
         public boolean callback(Pointer hWnd, Pointer userData) {
            byte[] windowText = new byte[512];
            user32.GetWindowTextA(hWnd, windowText, 512);
            String wText = Native.toString(windowText).trim();

            if (!wText.isEmpty() && wText.startsWith(startOfWindowName)) {
               return false;
            }
            return true;
         }
      }, null);
   }
   
   public static boolean windowExists(Pointer hWnd) {
      return user32.IsWindow(hWnd);
   }
   
   public static Pointer getWinHwnd(final String startOfWindowName) {
      callBackHwnd = null;
      
      user32.EnumWindows(new User32.WNDENUMPROC() {
         @Override
         public boolean callback(Pointer hWnd, Pointer userData) {
            byte[] windowText = new byte[512];
            user32.GetWindowTextA(hWnd, windowText, 512);
            String wText = Native.toString(windowText).trim();

            if (!wText.isEmpty() && wText.startsWith(startOfWindowName)) {
               callBackHwnd = hWnd;
               return false;
            }
            return true;
         }
      }, null);
      return callBackHwnd;
   }
   
   public static boolean setForegroundWindow(Pointer hWnd) {
      return user32.SetForegroundWindow(hWnd) != 0;
   }
   
   public static Pointer getForegroundWindow() {
      return user32.GetForegroundWindow();
   }
   
   public static String getForegroundWindowText() {
      Pointer hWnd = getForegroundWindow();
      int nMaxCount = 512;
      byte[] lpString = new byte[nMaxCount];
      int getWindowTextResult = user32.GetWindowTextA(hWnd, lpString, nMaxCount );
      if (getWindowTextResult == 0) {
         return "";
      }
      
      return Native.toString(lpString);
   }
   
   public static boolean isForegroundWindow(Pointer hWnd) {
      return user32.GetForegroundWindow().equals(hWnd);
   }
   
   public static boolean setForegroundWindow(String startOfWindowName) {
      Pointer hWnd = getWinHwnd(startOfWindowName);
      return user32.SetForegroundWindow(hWnd) != 0;
   }
   
   public static Rectangle getWindowRect(Pointer hWnd) throws JnaUtilException {
      if (hWnd == null) {
         throw new JnaUtilException("Failed to getWindowRect since Pointer hWnd is null");
      }
      Rectangle result = null;
      RECT rect = new RECT();
      boolean rectOK = user32.GetWindowRect(hWnd, rect);
      if (rectOK) {
         int x = rect.left;
         int y = rect.top;
         int width = rect.right - rect.left;
         int height = rect.bottom - rect.top;
         result = new Rectangle(x, y, width, height);
      }
      
      return result;
   }
   
   public static Rectangle getWindowRect(String startOfWindowName) throws JnaUtilException {
      Pointer hWnd = getWinHwnd(startOfWindowName);
      if (hWnd != null) {
         return getWindowRect(hWnd);
      } else {
         throw new JnaUtilException("Failed to getWindowRect for \"" + startOfWindowName + "\"");
      }
   }

   public static Pointer getWindow(Pointer hWnd, int uCmd) {
      return user32.GetWindow(hWnd, uCmd);
   }
   
   public static String getWindowText(Pointer hWnd) {
      int nMaxCount = 512;
      byte[] lpString = new byte[nMaxCount];
      int result = user32.GetWindowTextA(hWnd, lpString, nMaxCount);
      if (result == 0) {
         return "";
      }
      return Native.toString(lpString);
   }
   
   public static Pointer getOwnerWindow(Pointer hWnd) {
      return user32.GetWindow(hWnd, User32.GW_OWNER);
   }
   
   public static String getOwnerWindow(String childTitle) {
      Pointer hWnd = getWinHwnd(childTitle);
      Pointer parentHWnd = getOwnerWindow(hWnd);
      if (parentHWnd == null) {
         return "";
      }
      return getWindowText(parentHWnd);
      
   }

   public static void main(String[] args) throws InterruptedException {
      String[] testStrs = { "Untitled-Notepad", "Untitled - Notepad",
            "Untitled  -  Notepad", "Java-Epic", "Java - Epic", "Fubars rule!", 
            "The First Night", "New Tab", "Citrix X", "EHR PROD - SVC"};
      for (String testStr : testStrs) {
         Pointer hWnd = getWinHwnd(testStr);
         boolean isWindow = windowExists(hWnd);
         System.out.printf("%-22s %5b %16s %b%n", testStr, windowExists(testStr), hWnd, isWindow);
      }

      String ehrProd = "EHR PROD - SVC";
      Pointer hWnd = getWinHwnd(ehrProd);
      System.out.println("is it foreground window? " + isForegroundWindow(hWnd));
      boolean foo = setForegroundWindow(ehrProd);
      System.out.println("foregroundwindow: " + foo);
      Thread.sleep(400);
      System.out.println("is it foreground window? " + isForegroundWindow(hWnd));
      Thread.sleep(400);
      
      try {
         Rectangle rect = getWindowRect(ehrProd);
         Robot robot = new Robot();
         
         BufferedImage img = robot.createScreenCapture(rect);
         ImageIcon icon = new ImageIcon(img);
         JLabel label = new JLabel(icon);
         JOptionPane.showMessageDialog(null, label);
      
      } catch (AWTException e) {
         e.printStackTrace();
      } catch (JnaUtilException e) {
         e.printStackTrace();
      }
   }

}
jna.JnaUtilException.java
Java Code:
package jna;

public class JnaUtilException extends Exception {

   public JnaUtilException(String text) {
      super(text);
   }

}
pollForegroundWindow.PollForegroundWindow.java
Java Code:
package pollForegroundWindow;

import java.beans.PropertyChangeEvent;
import java.beans.PropertyChangeListener;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;

import javax.swing.event.SwingPropertyChangeSupport;

import com.sun.jna.Pointer;

import jna.JnaUtil;

public class PollForegroundWindow {
   public static final String FOREGROUND_WINDOW_TITLE = "foreground window title";
   private int frequency;
   private SwingPropertyChangeSupport spcSupport = new SwingPropertyChangeSupport(
         this);
   private ScheduledExecutorService scheduler;
   private ScheduledFuture<?> futureHandler;
   private String foregroundWindowTitle = "";
   
   public PollForegroundWindow(int frequency) {
      this.frequency = frequency;
   }
   
   public void startPolling() {
      Runnable myRunnable = new Runnable() {
         public void run() {
            String newTitle = JnaUtil.getForegroundWindowText();
            setForegroundWindowTitle(newTitle);
         }
      };
      
      TimeUnit timeUnit = TimeUnit.MILLISECONDS;
      scheduler = Executors.newScheduledThreadPool(1);
      long initialDelay = frequency;
      long period = frequency;
      futureHandler = scheduler.scheduleAtFixedRate(myRunnable, initialDelay,
            period, timeUnit);
   }
   
   public void stopPolling() {
      if (futureHandler != null && !futureHandler.isDone()) {
         futureHandler.cancel(true);
         scheduler.shutdownNow();
      }
   }
   
   private void setForegroundWindowTitle(String newTitle) {
      String oldValue = this.foregroundWindowTitle;
      String newValue = newTitle;
      this.foregroundWindowTitle = newValue ;
      
      spcSupport.firePropertyChange(FOREGROUND_WINDOW_TITLE, oldValue, newValue);
   }
   
   public String getForegroundWindowTitle() {
      return foregroundWindowTitle;
   }
   
   public void addPropertyChangeListener(PropertyChangeListener listener) {
      spcSupport.addPropertyChangeListener(listener);
   }
   
   public void removePropertyChangeListener(PropertyChangeListener listener) {
      spcSupport.removePropertyChangeListener(listener);
   }
   
   public static void main(String[] args) {
      int frequency = 50;
      PollForegroundWindow myForegroundPoller = new PollForegroundWindow(frequency);
      myForegroundPoller.addPropertyChangeListener(new PropertyChangeListener() {
         Pointer hWndPrevious = null;
         Pointer hWndCurrent = null;
         
         @Override
         public void propertyChange(PropertyChangeEvent evt) {
            if (PollForegroundWindow.FOREGROUND_WINDOW_TITLE.equals(evt.getPropertyName())) {
               System.out.println(evt.getNewValue());
               hWndPrevious = hWndCurrent;
               hWndCurrent = JnaUtil.getForegroundWindow();
               if (hWndPrevious == null || hWndCurrent == null) {
                  return;
               }
               if (hWndPrevious.equals(JnaUtil.getOwnerWindow(hWndCurrent))) {
                  System.out.println("\tOwned window found!");
               }
            }
         }
      });
      
      myForegroundPoller.startPolling();
   }

}
pollScrn.ScrnPollUtil.java
Java Code:
package pollScrn;

public class ScrnPollUtil {
   public static long calcDistanceSqrd(int[] biData1, int[] biData2) {
      if (biData1.length != biData2.length) {
         return -2L; // throw exception?
      }
      long result = 0L;
      for (int i = 0; i < biData1.length; i++) {
         int color1 = biData1[i];
         long r1 = (color1) & 0xFF;
         long g1 = (color1 >> 8) & 0xFF;
         long b1 = (color1 >> 16) & 0xFF;

         int color2 = biData2[i];
         long r2 = (color2) & 0xFF;
         long g2 = (color2 >> 8) & 0xFF;
         long b2 = (color2 >> 16) & 0xFF;

         long sqrLength = (r1 - r2) * (r1 - r2) + (g1 - g2) * (g1 - g2)
               + (b1 - b2) * (b1 - b2);
         result += sqrLength;
      }

      return result;
   }

   public static  long[] calcDistanceVector(int[] biData1, int[] biData2) {
      if (biData1.length != biData2.length) {
         String exceptionText = "biData1 and biData2 are different lengths: "
               + biData1.length + ", " + biData2.length;
         throw new IllegalArgumentException(exceptionText);
      }
      long[] result = new long[biData1.length];
      for (int i = 0; i < biData1.length; i++) {
         int color1 = biData1[i];
         long r1 = (color1) & 0xFF;
         long g1 = (color1 >> 8) & 0xFF;
         long b1 = (color1 >> 16) & 0xFF;

         int color2 = biData2[i];
         long r2 = (color2) & 0xFF;
         long g2 = (color2 >> 8) & 0xFF;
         long b2 = (color2 >> 16) & 0xFF;

         long sqrLength = (r1 - r2) * (r1 - r2) + (g1 - g2) * (g1 - g2)
               + (b1 - b2) * (b1 - b2);
         result[i] = sqrLength;
      }

      return result;
   }
   
   public static long[][] calcDistanceMatrix(long[] distVector, int scaledHeight, int scaledWidth) {
      long[][] matrix = new long[scaledHeight][scaledWidth];
      for (int row = 0; row < matrix.length; row++) {
         for (int col = 0; col < matrix[row].length; col++) {
            int index = row * scaledWidth + col;
            matrix[row][col] = distVector[index];
         }
      }

      return matrix;
   }
   
   public static long sumDistanceMatrix(long[][] distMatrix) {
      long sum = 0;
      for (long[] row : distMatrix) {
         for (long item : row) {
            sum += item;
         }
      }
      return sum;
   }
   
   public static long sumDistanceVector(long[] distVector) {
      long sum = 0;
      for (long i : distVector) {
         sum += i;
      }
      return sum;
   }
}
pollScrn.ScreenGrabber.java
Java Code:
package pollScrn;

import java.awt.AWTException;
import java.awt.Graphics;
import java.awt.Image;
import java.awt.Rectangle;
import java.awt.Robot;
import java.awt.image.BufferedImage;
import java.awt.image.DataBufferInt;

public class ScreenGrabber {
   private Robot robot;
   private Rectangle frameDim;
   private int scaledWidth;
   private int scaledHeight;
   private BufferedImage scaledBufferedImg;

   public ScreenGrabber(Rectangle frameDim, int scaledWidth, int scaledHeight)
         throws AWTException {
      robot = new Robot();
      this.frameDim = frameDim;
      this.scaledWidth = scaledWidth;
      this.scaledHeight = scaledHeight;
   }

   public int[] grabScreen() {
      BufferedImage img = robot.createScreenCapture(frameDim);
      Image scaledImg = img.getScaledInstance(scaledWidth, scaledHeight,
            BufferedImage.SCALE_AREA_AVERAGING);
      scaledBufferedImg = new BufferedImage(scaledWidth, scaledHeight,
            BufferedImage.TYPE_INT_ARGB);
      Graphics g = scaledBufferedImg.getGraphics();
      g.drawImage(scaledImg, 0, 0, null);
      g.dispose();
      int[] result = ((DataBufferInt) scaledBufferedImg.getRaster()
            .getDataBuffer()).getData();
      return result;
   }

   public Rectangle getFrameDim() {
      return frameDim;
   }

   public void setFrameDim(Rectangle frameDim) {
      this.frameDim = frameDim;
   }

   public int getScaledWidth() {
      return scaledWidth;
   }

   public void setScaledWidth(int scaledWidth) {
      this.scaledWidth = scaledWidth;
   }

   public int getScaledHeight() {
      return scaledHeight;
   }

   public void setScaledHeight(int scaledHeight) {
      this.scaledHeight = scaledHeight;
   }

   public BufferedImage getScaledBufferedImg() {
      return scaledBufferedImg;
   }
}
pollScrn.ScreenImagePoller.java
Java Code:
import java.awt.AWTException;
import java.awt.Rectangle;
import java.awt.image.BufferedImage;
import java.beans.PropertyChangeListener;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;

import javax.swing.event.SwingPropertyChangeSupport;

import jna.JnaUtil;

import com.sun.jna.Pointer;

//TODO: concurrency!
public class ScreenImagePoller {

   public static final String DELTA_DIST_SQRD = "delta dist sqrd";
   public static final String DELTA_DIST_VECTOR = "delta dist vector";
   public static final String DELTA_DIST_MATRIX = "delta dist matrix";
   private int[] currentImgDataVector;
   private int[] referenceImgDataVector;
   private long[] distanceVector;
   private long[][] distanceMatrix;
   private long deltaDistSqrd = 0L;
   private long deltaThreshold = 0L;
   private long edgeThreshold = 0;
   private Rectangle frameDim;
   private int freq;
   private int scaledHeight;
   private int scaledWidth;
   private SwingPropertyChangeSupport spcSupport = new SwingPropertyChangeSupport(
         this);
   private ScheduledExecutorService scheduler;
   private ScheduledFuture<?> futureHandler;
   private ScreenGrabber screenGrabber;
   private Pointer hWnd = null;
   private volatile boolean reset = false;

   public ScreenImagePoller(Rectangle frameDim, int scaledWidth,
         int scaledHeight, int freq) throws AWTException {
      this.frameDim = frameDim;
      this.scaledWidth = scaledWidth;
      this.scaledHeight = scaledHeight;
      this.freq = freq;
      // robot = new Robot();
      screenGrabber = new ScreenGrabber(frameDim, scaledWidth, scaledHeight);
   }

   public void setScreenGrabber(ScreenGrabber screenGrabber) {
      this.screenGrabber = screenGrabber;
   }

   public void addPropertyChangeListener(PropertyChangeListener listener) {
      spcSupport.addPropertyChangeListener(listener);
   }

   public int[] getCurrentImgDataVector() {
      return currentImgDataVector;
   }

   public long getDeltaDistSqrd() {
      return deltaDistSqrd;
   }

   public long getDeltaThreshold() {
      return deltaThreshold;
   }

   public void setEdgeThreshold(int edgeThreshold) {
      this.edgeThreshold = edgeThreshold;
   }

   public long getEdgeThreshold() {
      return edgeThreshold;
   }

   public long[] getDistanceVector() {
      return distanceVector;
   }

   public Rectangle getFrameDim() {
      return frameDim;
   }

   public int getFreq() {
      return freq;
   }

   public int[] getReferenceImgDataVector() {
      return referenceImgDataVector;
   }

   public int getScaledHeight() {
      return scaledHeight;
   }

   public int getScaledWidth() {
      return scaledWidth;
   }

   public void startPolling() {
      final Runnable myRunnable = new Runnable() {
         public void run() {
            if (hWnd != null) {
               // only poll if my window is in the foreground
               if (!JnaUtil.windowExists(hWnd)) {
                  return;
               }
               if (!JnaUtil.isForegroundWindow(hWnd)) {
                  Pointer foreGroundHwnd = JnaUtil.getForegroundWindow();
                  if (!hWnd.equals(JnaUtil.getOwnerWindow(foreGroundHwnd))) {
                     return;
                  }
               }
            }

            currentImgDataVector = screenGrabber.grabScreen();
            if (referenceImgDataVector == null) {
               referenceImgDataVector = currentImgDataVector;
               return;
            }

            synchronized (this) {
               long[] distVector = ScrnPollUtil.calcDistanceVector(
                     referenceImgDataVector, currentImgDataVector);
               setDistanceVector(distVector);

               long[][] distMatrix = ScrnPollUtil.calcDistanceMatrix(
                     distVector, scaledHeight, scaledWidth);
               setDistanceMatrix(distMatrix);
               // synchronized (this) {
               if (reset) {
                  referenceImgDataVector = currentImgDataVector;
                  // reset = false;
                  setReset(false);
               }
            }

         }
      };
      long initialDelay = 0;
      long period = freq;
      TimeUnit timeUnit = TimeUnit.MILLISECONDS;
      scheduler = Executors.newScheduledThreadPool(1);
      futureHandler = scheduler.scheduleAtFixedRate(myRunnable, initialDelay,
            period, timeUnit);

   }

   public void stopPolling() {
      if (futureHandler != null && !futureHandler.isDone()) {
         futureHandler.cancel(true);
         scheduler.shutdownNow();
      }
   }

   public void removePropertyChangeListener(PropertyChangeListener listener) {
      spcSupport.removePropertyChangeListener(listener);
   }

   public void setDeltaDistSqrd(long deltDistSqrd) {
      long oldValue = this.deltaDistSqrd;
      long newValue = deltDistSqrd;

      this.deltaDistSqrd = newValue;

      if (newValue > deltaThreshold) {
         spcSupport.firePropertyChange(DELTA_DIST_SQRD, oldValue, newValue);
      }
   }

   public void setDistanceVector(long[] distanceVector) {
      long[] oldValue = this.distanceVector;
      long[] newValue = distanceVector;
      this.distanceVector = newValue;
      spcSupport.firePropertyChange(DELTA_DIST_VECTOR, oldValue, newValue);
   }

   public long[][] getDistanceMatrix() {
      return distanceMatrix;
   }

   public void setDistanceMatrix(long[][] distanceMatrix) {
      long[][] oldValue = this.distanceMatrix;
      long[][] newValue = distanceMatrix;
      this.distanceMatrix = newValue;
      spcSupport.firePropertyChange(DELTA_DIST_MATRIX, oldValue, newValue);

   }

   public Pointer getPointer() {
      return hWnd;
   }

   public void setPointer(Pointer pointer) {
      this.hWnd = pointer;
   }

   public boolean isReset() {
      return reset;
   }

   public synchronized void setReset(boolean reset) {
      this.reset = reset;
   }

   public BufferedImage getCurrentScaledImg() {
      return screenGrabber.getScaledBufferedImg();
   }

}

Submit "Windows Screen Grabber JNA methods that I'm working on" to Facebook Submit "Windows Screen Grabber JNA methods that I'm working on" to Digg Submit "Windows Screen Grabber JNA methods that I'm working on" to del.icio.us Submit "Windows Screen Grabber JNA methods that I'm working on" to StumbleUpon Submit "Windows Screen Grabber JNA methods that I'm working on" to Google

Updated 05-26-2012 at 05:56 AM by Fubarable

Tags: -1' Add / Edit Tags
Categories
Uncategorized

Comments

  1. Fubarable's Avatar
    • |
    • permalink
    OK, how the heck to I edit the original post?