benchmarks/Java/IrrigationController/MotionDetection.java

   1 package IrrigationController;
   2 // IoT packages
   3 import iotcode.interfaces.*;
   4
   5 // BoofCv packages
   6 import boofcv.alg.background.BackgroundModelStationary;
   7 import boofcv.factory.background.ConfigBackgroundGaussian;
   8 import boofcv.factory.background.FactoryBackgroundModel;
   9 import boofcv.gui.binary.VisualizeBinaryData;
  10 import boofcv.gui.image.ImageGridPanel;
  11 import boofcv.gui.image.ShowImages;
  12 import boofcv.io.MediaManager;
  13 import boofcv.io.UtilIO;
  14 import boofcv.io.image.SimpleImageSequence;
  15 import boofcv.io.image.ConvertBufferedImage;
  16 import boofcv.io.wrapper.DefaultMediaManager;
  17 import boofcv.struct.image.ImageBase;
  18 import boofcv.struct.image.ImageFloat32;
  19 import boofcv.struct.image.ImageType;
  20 import boofcv.struct.image.ImageUInt8;
  21 import boofcv.alg.filter.blur.BlurImageOps;
  22
  23
  24 // Standard Java Packages
  25 import java.awt.image.BufferedImage;
  26 import java.util.Date;
  27 import java.util.concurrent.locks.Lock;
  28 import java.util.concurrent.locks.ReadWriteLock;
  29 import java.util.concurrent.locks.ReentrantReadWriteLock;
  30 import java.util.concurrent.atomic.AtomicBoolean;
  31 import java.awt.image.ColorModel;
  32 import java.awt.image.WritableRaster;
  33 import java.util.List;
  34 import java.util.ArrayList;
  35 import java.io.ByteArrayInputStream;
  36 import java.io.ByteArrayOutputStream;
  37 import java.io.File;
  38 import java.io.IOException;
  39 import java.io.InputStream;
  40 import javax.imageio.ImageIO;
  41
  42 // RMI Packages
  43 import java.rmi.RemoteException;
  44 import java.rmi.server.UnicastRemoteObject;
  45
  46 // Checker annotations
  47 //import iotchecker.qual.*;
  48
  49
  50
  51 /** Class MotionDetection to do motion detection using images
  52  *
  53  *
  54  * @author      Ali Younis <ayounis @ uci.edu>
  55  * @version     1.0
  56  * @since       2016-03-21
  57  */
  58 class MotionDetection extends UnicastRemoteObject implements CameraCallback {
  59
  60         // Define Like variables
  61         private static boolean DO_GRAPHICAL_USER_INTERFACE = false;
  62
  63         /*******************************************************************************************************************************************
  64         **
  65         **  Constants
  66         **
  67         *******************************************************************************************************************************************/
  68         private final float MOTION_DETECTED_THRESHOLD_PERCENTAGE = 5;
  69
  70
  71         /*******************************************************************************************************************************************
  72         **
  73         **  Variables
  74         **
  75         *******************************************************************************************************************************************/
  76
  77         // Timestamp buffer and locks needed for that safety on that buffer
  78         // This is the buffer for post-detection algorithm use
  79         private Date timestampOfLastMotion = null;
  80         private ReadWriteLock timestampReadWriteLock = new ReentrantReadWriteLock();
  81         private Lock timestampReadLock = timestampReadWriteLock.readLock();
  82         private Lock timestampWriteLock = timestampReadWriteLock.writeLock();
  83
  84         // Flag for when new data is available and ready for processing
  85         private AtomicBoolean newFrameAvailable = new AtomicBoolean(false);
  86
  87         // Flag for determining if motion has been detected and therefore
  88         // the callbacks should be issued
  89         private AtomicBoolean motionDetected = new AtomicBoolean(false);
  90
  91         // Image and timestamp buffers and  locks needed for that safety on those buffers
  92         // Timestamp buffer for pre-detection algorithm use
  93         private BufferedImage latestImage = null;
  94         private Date possibleDate = null;
  95         private ReadWriteLock imageReadWriteLock = new ReentrantReadWriteLock();
  96         private Lock imageReadLock = imageReadWriteLock.readLock();
  97         private Lock imageWriteLock = imageReadWriteLock.writeLock();
  98
  99         // List of objects wishing to receive callbacks from this class.
 100         private List<MotionDetectionCallback> callbackList = new ArrayList<MotionDetectionCallback>();
 101
 102         // Variables to help with motion detection
 103         private ConfigBackgroundGaussian configGaussian = null;
 104         private BackgroundModelStationary backgroundDetector = null;
 105         private ImageUInt8 segmented = null;
 106         private ImageFloat32 newFrameFloat = null;
 107
 108         // counts the number of frames since a background image is added to algorithm
 109         private int frameCounter = 0;
 110
 111
 112
 113         /*******************************************************************************************************************************************
 114         **
 115         **  Threads
 116         **
 117         *******************************************************************************************************************************************/
 118         private Thread workThread = null;
 119         private Thread callBackThread = null;
 120
 121         /*******************************************************************************************************************************************
 122         **
 123         **  GUI Stuff (Used Only for Testing)
 124         **
 125         *******************************************************************************************************************************************/
 126         ImageGridPanel gui;
 127
 128
 129
 130         /** Constructor
 131          *
 132          *   @param _threshold       [float], Variable for gaussian background detector.
 133          *   @param _learnSpeed      [float], Variable for gaussian background detector.
 134          *   @param _initialVariance [float], Variable for gaussian background detector.
 135          *   @param _minDifference   [float], Variable for gaussian background detector.
 136          *
 137          */
 138         public MotionDetection(float _threshold, float _learnSpeed, float _initialVariance, float _minDifference) throws RemoteException {
 139
 140                 // Configure the Gaussian model used for background detection
 141                 configGaussian = new ConfigBackgroundGaussian(_threshold, _learnSpeed);
 142                 configGaussian.initialVariance = _initialVariance;
 143                 configGaussian.minimumDifference = _minDifference;
 144
 145                 // setup the background detector
 146                 ImageType imageType = ImageType.single(ImageFloat32.class);
 147                 backgroundDetector = FactoryBackgroundModel.stationaryGaussian(configGaussian, imageType);
 148
 149                 // setup the gui if we are going to use it
 150                 if (DO_GRAPHICAL_USER_INTERFACE) {
 151
 152                         // create an image grid for images to place on, tile fashion
 153                         gui = new ImageGridPanel(1, 2);
 154
 155                         // make the window large so we dont have to manually resize with the mouse
 156                         gui.setSize(1920, 1080);
 157
 158                         // Make the window visible and set the title
 159                         ShowImages.showWindow(gui, "Static Scene: Background Segmentation", true);
 160                 }
 161
 162                 // Launch the worker thread
 163                 workThread = new Thread(new Runnable() {
 164                         public void run() {
 165
 166                                 while (true) {
 167                                         runMotionDetection();
 168                                 }
 169                         }
 170                 });
 171                 workThread.start();
 172
 173
 174                 // Launch the callback thread
 175                 callBackThread = new Thread(new Runnable() {
 176                         public void run() {
 177
 178                                 while (true) {
 179                                         doCallbacks();
 180                                 }
 181                         }
 182                 });
 183                 callBackThread.start();
 184         }
 185
 186
 187         /*******************************************************************************************************************************************
 188         **
 189         **  Public Methods
 190         **
 191         *******************************************************************************************************************************************/
 192
 193         /** Method to add a new frame to the motion detector.
 194          *
 195          *   @param _newFrame  [byte[]], Frame data of new frame.
 196          *   @param _timestamp [Date]  , Timestamp of new frame.
 197          *
 198          *   @return [void] None.
 199          */
 200         public void addFrame(byte[]  _newFrame, Date _timestamp) {
 201                 BufferedImage img = null;
 202
 203                 try {
 204                         // Parse the byte array into a Buffered Image
 205                         InputStream in = new ByteArrayInputStream(_newFrame);
 206                         img = ImageIO.read(in);
 207
 208                 } catch (Exception e) {
 209                         e.printStackTrace();
 210                         return;
 211                 }
 212
 213                 // Save the image and timestamp for use later
 214                 imageWriteLock.lock();                                          // lock the image and timestamp buffers since multithread
 215                 latestImage = img;                                                              // image into image buffer
 216                 possibleDate = _timestamp;                              // timestamp into timestamp buffer
 217                 imageWriteLock.unlock();                                        // Never forget to unlock
 218
 219                 // flag the worker thread that there is new data ready for processing
 220                 newFrameAvailable.set(true);
 221         }
 222
 223         /** Method to get the timestamp of the last time motion was detected
 224          *
 225          *   @return [Date] timestamp of last motion or null if no motion was ever detected.
 226          */
 227         public long getTimestampOfLastMotion() {
 228                 Date ret = null;
 229
 230                 // Be safe because multithread
 231                 timestampReadLock.lock();
 232
 233                 // Checks if there was ever motion, if not then timestampOfLastMotion
 234                 // will be null
 235                 if (timestampOfLastMotion != null) {
 236                         // Clone since we don't know what the other person is going to do
 237                         // with the timestamp
 238                         ret = (Date)timestampOfLastMotion.clone();
 239                 }
 240
 241                 timestampReadLock.unlock();
 242                 long retLong = ret.getTime();
 243
 244                 return retLong;
 245         }
 246
 247
 248         /** Method to add a new frame to the motion detector from a camera
 249          *
 250          *   @param _camera  [Camera], Camera that has the new data.
 251          *
 252          *   @return [void] None.
 253          */
 254         //public void newCameraFrameAvailable(@NonLocalRemote Camera _camera) {
 255         public void newCameraFrameAvailable(byte[] latestFrame, long timeStamp) {
 256                 BufferedImage img = null;
 257
 258                 try {
 259                         // Parse the byte array into a Buffered Image
 260                         InputStream in = new ByteArrayInputStream(latestFrame);
 261                         img = ImageIO.read(in);
 262
 263                 } catch (RemoteException e) {
 264                         e.printStackTrace();
 265                         return;
 266
 267                 } catch (Exception e) {
 268                         e.printStackTrace();
 269                         return;
 270
 271                 }
 272
 273                 // Save the image and timestamp for use later
 274                 imageWriteLock.lock();                  // lock the image and timestamp buffers since multithread
 275                 latestImage = img;                              // image into image buffer
 276
 277                 // timestamp from camera into timestamo buffer
 278                 long dateLong = timeStamp;
 279                 possibleDate = new Date(dateLong);
 280
 281                 imageWriteLock.unlock();                // Never forget to unlock
 282
 283                 // flag the worker thread that there is new data ready for processing
 284                 newFrameAvailable.set(true);
 285         }
 286
 287         /** Method to register an object to recieve callbacks from this motion detector
 288          *
 289          *   @param _mdc  [MotionDetectionCallback], object to recieve callbacks.
 290          *
 291          *   @return [void] None.
 292          */
 293         public void registerCallback(MotionDetectionCallback _mdc) {
 294                 callbackList.add(_mdc);
 295         }
 296
 297         /*******************************************************************************************************************************************
 298         **
 299         **  Helper Methods
 300         **
 301         *******************************************************************************************************************************************/
 302
 303         /** Method that constantly loops checking if new data is available.  If there is
 304          *   new data, it is processed.
 305          *   This method should be run on a separate thread.
 306          *
 307          *   @return [void] None.
 308          */
 309         private void runMotionDetection() {
 310
 311                 // check if there is a new frame availble, only runs detection if there is new data to save
 312                 // computation time
 313                 if (!newFrameAvailable.get()) {
 314                         return;
 315                 }
 316
 317                 // Lock since we are accessing the data buffers
 318                 imageReadLock.lock();
 319
 320                 // processing data so now the buffered data is old
 321                 newFrameAvailable.set(false);
 322
 323                 // copy from buffer to local for processing
 324                 Date tmpDate = possibleDate;
 325
 326                 // Allocate space for the segmented image based on the first image we received
 327                 // cannot pre-allocate this since we do not know what the size of the images is
 328                 // before the first image arrives
 329                 if (segmented == null) {
 330                         segmented = new ImageUInt8(latestImage.getWidth(), latestImage.getHeight());
 331                 }
 332
 333                 // copy from data buffers and convert into correct data type for BoofCv libraries
 334                 newFrameFloat = ConvertBufferedImage.convertFrom(latestImage, newFrameFloat);
 335
 336                 // All done accessing the data buffers
 337                 imageReadLock.unlock();
 338
 339                 // Run background detection
 340                 backgroundDetector.segment(newFrameFloat, segmented);
 341
 342                 // Update the background baseline every 10 frames, helps the algorithm
 343                 frameCounter++;
 344                 if (frameCounter > 10) {
 345                         backgroundDetector.updateBackground(newFrameFloat);
 346                         frameCounter = 0;
 347                 }
 348
 349                 // get the raw pixel data, gray-scale image
 350                 byte[] frameData = segmented.getData();
 351
 352                 // count the number of pixels of the image that was deemed as "motion"
 353                 double count = 0;
 354                 double countMotion = 0;
 355                 for (byte b : frameData) {
 356                         count++;
 357                         if (b > 0) {
 358                                 countMotion++;
 359                         }
 360                 }
 361
 362                 // calculate the percentage of the image that was in motion
 363                 double percentMotion = (countMotion / count) * 100.0;
 364
 365                 // Check if a high enough percentage of the image was in motion to say that there was motion in this frame of data
 366                 if (percentMotion > MOTION_DETECTED_THRESHOLD_PERCENTAGE) {
 367
 368                         // Motion detected so save timestamp of this frame to another buffer
 369                         timestampWriteLock.lock();
 370                         timestampOfLastMotion = (Date)tmpDate.clone();                  // clone to a different buffer
 371                         timestampWriteLock.unlock();
 372
 373                         System.out.println("Motion Detected (with percentage: " + Double.toString(percentMotion) + "%)");
 374                 }
 375
 376                 // Do output to the screen if we are using gui mode
 377                 if (DO_GRAPHICAL_USER_INTERFACE) {
 378
 379                         // change image data unto correct type for rendering
 380                         BufferedImage visualized1 = new BufferedImage(segmented.width, segmented.height, BufferedImage.TYPE_INT_RGB);
 381                         VisualizeBinaryData.renderBinary(segmented, false, visualized1);
 382
 383                         // change image data unto correct type for rendering
 384                         BufferedImage visualized2 = null;
 385                         visualized2 = ConvertBufferedImage.convertTo(newFrameFloat, visualized2, true);
 386
 387                         // place the images into the image grid
 388                         gui.setImage(0, 1, visualized1);
 389                         gui.setImage(0, 2, visualized2);
 390
 391                         // trigger rendering
 392                         gui.repaint();
 393                 }
 394         }
 395
 396         /** Method that constantly loops checking if the callbacks should be issues and
 397          *   issues the callbacks if they should be issues.
 398          *   This method should be run on a separate thread.
 399          *
 400          *   @return [void] None.
 401          */
 402         private void doCallbacks() {
 403
 404                 // Keep looping forever for callback
 405                 while (true) {
 406
 407                         // If motion detected
 408                         if (motionDetected.compareAndSet(true, false)) {
 409
 410                                 // Motion was detected so issue callbacks to all objects that registered
 411                                 // to receive callback from this class.
 412                                 for (MotionDetectionCallback c : callbackList) {
 413                                         c.motionDetected(this.getTimestampOfLastMotion());
 414                                 }
 415
 416                         } else {
 417
 418                                 // Sleep for 15 millisec to give time for new frame to arrive
 419                                 try {
 420                                         Thread.sleep(15);
 421                                 } catch (InterruptedException ie) {
 422                                 }
 423                         }
 424                 }
 425         }
 426 }
 427