benchmarks/Java/SmartLightsController/MotionDetection.java

   1 package SmartLightsController;
   2
   3 // IoT packages
   4 import iotcode.annotation.*;
   5 import iotcode.interfaces.*;
   6
   7 // BoofCv packages
   8 import boofcv.alg.background.BackgroundModelStationary;
   9 import boofcv.factory.background.ConfigBackgroundGaussian;
  10 import boofcv.factory.background.FactoryBackgroundModel;
  11 import boofcv.gui.binary.VisualizeBinaryData;
  12 import boofcv.gui.image.ImageGridPanel;
  13 import boofcv.gui.image.ShowImages;
  14 import boofcv.io.MediaManager;
  15 import boofcv.io.UtilIO;
  16 import boofcv.io.image.SimpleImageSequence;
  17 import boofcv.io.image.ConvertBufferedImage;
  18 import boofcv.io.wrapper.DefaultMediaManager;
  19 import boofcv.struct.image.ImageBase;
  20 import boofcv.struct.image.ImageFloat32;
  21 import boofcv.struct.image.ImageType;
  22 import boofcv.struct.image.ImageUInt8;
  23 import boofcv.alg.filter.blur.BlurImageOps;
  24
  25
  26 // Standard Java Packages
  27 import java.awt.image.BufferedImage;
  28 import java.util.Date;
  29 import java.util.concurrent.locks.Lock;
  30 import java.util.concurrent.locks.ReadWriteLock;
  31 import java.util.concurrent.locks.ReentrantReadWriteLock;
  32 import java.util.concurrent.atomic.AtomicBoolean;
  33 import java.awt.image.ColorModel;
  34 import java.awt.image.WritableRaster;
  35 import java.util.List;
  36 import java.util.ArrayList;
  37 import java.io.ByteArrayInputStream;
  38 import java.io.ByteArrayOutputStream;
  39 import java.io.File;
  40 import java.io.IOException;
  41 import java.io.InputStream;
  42 import javax.imageio.ImageIO;
  43
  44 // RMI Packages
  45 import java.rmi.RemoteException;
  46
  47 // For testing
  48 import java.net.*;
  49
  50 import java.rmi.Remote;
  51 import java.rmi.RemoteException;
  52
  53 import java.util.Iterator;
  54
  55 // Checker annotations
  56 //import iotchecker.qual.*;
  57
  58 /** Class MotionDetection to do motion detection using images
  59  *
  60  *
  61  * @author      Ali Younis <ayounis @ uci.edu>
  62  * @version     1.0
  63  * @since       2016-03-21
  64  */
  65 class MotionDetection implements CameraCallback {
  66
  67         // Define Like variables
  68         private static boolean DO_GRAPHICAL_USER_INTERFACE = false;
  69
  70         /*******************************************************************************************************************************************
  71         **
  72         **  Constants
  73         **
  74         *******************************************************************************************************************************************/
  75         private final float MOTION_DETECTED_THRESHOLD_PERCENTAGE = 15;
  76
  77
  78         /*******************************************************************************************************************************************
  79         **
  80         **  Variables
  81         **
  82         *******************************************************************************************************************************************/
  83
  84         // Timestamp buffer and locks needed for that safety on that buffer
  85         // This is the buffer for post-detection algorithm use
  86         private Date timestampOfLastMotion = null;
  87         private ReadWriteLock timestampReadWriteLock = new ReentrantReadWriteLock();
  88         private Lock timestampReadLock = timestampReadWriteLock.readLock();
  89         private Lock timestampWriteLock = timestampReadWriteLock.writeLock();
  90
  91         // Flag for when new data is available and ready for processing
  92         private AtomicBoolean newFrameAvailable = new AtomicBoolean(false);
  93
  94         // Flag for determining if motion has been detected and therefore
  95         // the callbacks should be issued
  96         private AtomicBoolean motionDetected = new AtomicBoolean(false);
  97
  98         // Image and timestamp buffers and  locks needed for that safety on those buffers
  99         // Timestamp buffer for pre-detection algorithm use
 100         private BufferedImage latestImage = null;
 101         private Date possibleDate = null;
 102         private ReadWriteLock imageReadWriteLock = new ReentrantReadWriteLock();
 103         private Lock imageReadLock = imageReadWriteLock.readLock();
 104         private Lock imageWriteLock = imageReadWriteLock.writeLock();
 105
 106         // List of objects wishing to receive callbacks from this class.
 107         private List<MotionDetectionCallback>
 108         callbackList = new ArrayList<MotionDetectionCallback>();
 109
 110         // Variables to help with motion detection
 111         private ConfigBackgroundGaussian configGaussian = null;
 112         private BackgroundModelStationary backgroundDetector = null;
 113         private ImageUInt8 segmented = null;
 114         private ImageFloat32 newFrameFloat = null;
 115
 116         // counts the number of frames since a background image is added to algorithm
 117         private int frameCounter = 0;
 118
 119         private CameraSmart _camera;
 120
 121         /*******************************************************************************************************************************************
 122         **
 123         **  Threads
 124         **
 125         *******************************************************************************************************************************************/
 126         private Thread workThread = null;
 127         private Thread callBackThread = null;
 128
 129         /*******************************************************************************************************************************************
 130         **
 131         **  GUI Stuff (Used Only for Testing)
 132         **
 133         *******************************************************************************************************************************************/
 134         ImageGridPanel gui;
 135
 136         /** Constructor
 137          *
 138          *   @param _threshold       [float], Variable for gaussian background detector.
 139          *   @param _learnSpeed      [float], Variable for gaussian background detector.
 140          *   @param _initialVariance [float], Variable for gaussian background detector.
 141          *   @param _minDifference   [float], Variable for gaussian background detector.
 142          *
 143          */
 144         public MotionDetection(float _threshold, float _learnSpeed, float _initialVariance, float _minDifference)
 145         throws RemoteException {
 146
 147                 // Configure the Gaussian model used for background detection
 148                 configGaussian = new ConfigBackgroundGaussian(_threshold, _learnSpeed);
 149                 configGaussian.initialVariance = _initialVariance;
 150                 configGaussian.minimumDifference = _minDifference;
 151
 152                 // setup the background detector
 153                 ImageType imageType = ImageType.single(ImageFloat32.class);
 154                 backgroundDetector = FactoryBackgroundModel.stationaryGaussian(configGaussian, imageType);
 155
 156                 // setup the gui if we are going to use it
 157                 if (DO_GRAPHICAL_USER_INTERFACE) {
 158
 159                         // create an image grid for images to place on, tile fashion
 160                         gui = new ImageGridPanel(1, 2);
 161
 162                         // make the window large so we dont have to manually resize with the mouse
 163                         gui.setSize(1920, 1080);
 164
 165                         // Make the window visible and set the title
 166                         ShowImages.showWindow(gui, "Static Scene: Background Segmentation", true);
 167                 }
 168
 169                 // Launch the worker thread
 170                 workThread = new Thread(new Runnable() {
 171                         public void run() {
 172
 173                                 while (true) {
 174                                         runMotionDetection();
 175                                 }
 176                         }
 177                 });
 178                 workThread.start();
 179
 180
 181                 // Launch the callback thread
 182                 callBackThread = new Thread(new Runnable() {
 183                         public void run() {
 184
 185                                 while (true) {
 186                                         doCallbacks();
 187                                 }
 188                         }
 189                 });
 190                 callBackThread.start();
 191         }
 192
 193
 194         /*******************************************************************************************************************************************
 195         **
 196         **  Public Methods
 197         **
 198         *******************************************************************************************************************************************/
 199
 200         /** Method to add a new frame to the motion detector.
 201          *
 202          *   @param _newFrame  [byte[]], Frame data of new frame.
 203          *   @param _timestamp [Date]  , Timestamp of new frame.
 204          *
 205          *   @return [void] None.
 206          */
 207         public void addFrame(byte[]  _newFrame, Date _timestamp) {
 208                 BufferedImage img = null;
 209
 210                 try {
 211                         // Parse the byte array into a Buffered Image
 212                         InputStream in = new ByteArrayInputStream(_newFrame);
 213                         img = ImageIO.read(in);
 214
 215                 } catch (Exception e) {
 216                         e.printStackTrace();
 217                         return;
 218                 }
 219
 220                 // Save the image and timestamp for use later
 221                 imageWriteLock.lock();// lock the image and timestamp buffers since multithread
 222                 latestImage = img;// image into image buffer
 223                 possibleDate = _timestamp;// timestamp into timestamp buffer
 224                 imageWriteLock.unlock();// Never forget to unlock
 225
 226                 // flag the worker thread that there is new data ready for processing
 227                 newFrameAvailable.set(true);
 228         }
 229
 230         /** Method to get the timestamp of the last time motion was detected
 231          *
 232          *   @return [Date] timestamp of last motion or null if no motion was ever detected.
 233          */
 234         public Date getTimestampOfLastMotion() {
 235                 Date ret = null;
 236
 237                 // Be safe because multithread
 238                 timestampReadLock.lock();
 239
 240                 // Checks if there was ever motion, if not then timestampOfLastMotion
 241                 // will be null
 242                 if (timestampOfLastMotion != null) {
 243                         // Clone since we don't know what the other person is going to do
 244                         // with the timestamp
 245                         ret = (Date)timestampOfLastMotion.clone();
 246                 }
 247
 248                 timestampReadLock.unlock();
 249
 250                 return ret;
 251         }
 252
 253
 254         /** Method to add a new frame to the motion detector from a camera
 255          *
 256          *   @param _camera  [Camera], Camera that has the new data.
 257          *
 258          *   @return [void] None.
 259          */
 260         public void newCameraFrameAvailable(byte[] latestFrame, long timeStamp) {
 261         //public void newCameraFrameAvailable(CameraSmart _camera) {
 262                 BufferedImage img = null;
 263
 264                 //byte[] newImg = _camera.getLatestFrame();
 265                 try {
 266                         //InputStream in = new ByteArrayInputStream(_camera.getLatestFrame());
 267                         //InputStream in = new ByteArrayInputStream(newImg);
 268                         // Parse the byte array into a Buffered Image
 269                         InputStream in = new ByteArrayInputStream(latestFrame);
 270                         img = ImageIO.read(in);
 271
 272                 } catch (RemoteException e) {
 273                         e.printStackTrace();
 274                         return;
 275
 276                 } catch (Exception e) {
 277                         e.printStackTrace();
 278                         return;
 279
 280                 }
 281
 282                 // Save the image and timestamp for use later
 283                 imageWriteLock.lock();  // lock the image and timestamp buffers since multithread
 284                 latestImage = img;              // image into image buffer
 285
 286                 // timestamp from camera into timestamp buffer
 287                 long dateLong = timeStamp;
 288                 //long dateLong = _camera.getTimestamp();
 289                 //System.out.println("dateLong: " + dateLong + " - newImage size: " + newImg.length);
 290                 possibleDate = new Date(dateLong);
 291
 292                 imageWriteLock.unlock();        // Never forget to unlock
 293
 294                 // flag the worker thread that there is new data ready for processing
 295                 newFrameAvailable.set(true);
 296         }
 297
 298         /** Method to register an object to recieve callbacks from this motion detector
 299          *
 300          *   @param _mdc  [MotionDetectionCallback], object to recieve callbacks.
 301          *
 302          *   @return [void] None.
 303          */
 304         public void registerCallback(MotionDetectionCallback _mdc) {
 305                 callbackList.add(_mdc);
 306         }
 307
 308         /*******************************************************************************************************************************************
 309         **
 310         **  Helper Methods
 311         **
 312         *******************************************************************************************************************************************/
 313
 314         /** Method that constantly loops checking if new data is available.  If there is
 315          *   new data, it is processed.
 316          *   This method should be run on a separate thread.
 317          *
 318          *   @return [void] None.
 319          */
 320         private void runMotionDetection() {
 321
 322                 // check if there is a new frame availble, only runs detection if there is new data to save
 323                 // computation time
 324                 if (!newFrameAvailable.get()) {
 325                         return;
 326                 }
 327
 328                 // Lock since we are accessing the data buffers
 329                 imageReadLock.lock();
 330
 331                 // processing data so now the buffered data is old
 332                 newFrameAvailable.set(false);
 333
 334                 // copy from buffer to local for processing
 335                 Date tmpDate = possibleDate;
 336
 337                 // Allocate space for the segmented image based on the first image we received
 338                 // cannot pre-allocate this since we do not know what the size of the images is
 339                 // before the first image arrives
 340                 if (segmented == null) {
 341                         segmented = new ImageUInt8(latestImage.getWidth(), latestImage.getHeight());
 342                 }
 343
 344                 // copy from data buffers and convert into correct data type for BoofCv libraries
 345                 newFrameFloat = ConvertBufferedImage.convertFrom(latestImage, newFrameFloat);
 346
 347                 // All done accessing the data buffers
 348                 imageReadLock.unlock();
 349
 350                 // Run background detection
 351                 backgroundDetector.segment(newFrameFloat, segmented);
 352
 353                 // Update the background baseline every 10 frames, helps the algorithm
 354                 frameCounter++;
 355                 if (frameCounter > 10) {
 356                         backgroundDetector.updateBackground(newFrameFloat);
 357                         frameCounter = 0;
 358                 }
 359
 360                 // get the raw pixel data, gray-scale image
 361                 byte[] frameData = segmented.getData();
 362
 363                 // count the number of pixels of the image that was deemed as "motion"
 364                 double count = 0;
 365                 double countMotion = 0;
 366                 for (byte b : frameData) {
 367                         count++;
 368                         if (b > 0) {
 369                                 countMotion++;
 370                         }
 371                 }
 372
 373                 // calculate the percentage of the image that was in motion
 374                 double percentMotion = (countMotion / count) * 100.0;
 375                 //System.out.println("countMotion: " + countMotion);
 376                 //System.out.println("count: " + count);
 377                 //System.out.println("Percent motion: " + percentMotion);
 378
 379                 // Check if a high enough percentage of the image was in motion to say that there was motion in this frame of data
 380                 if (percentMotion > MOTION_DETECTED_THRESHOLD_PERCENTAGE) {
 381
 382                         // Motion detected so save timestamp of this frame to another buffer
 383                         timestampWriteLock.lock();
 384                         timestampOfLastMotion = (Date)tmpDate.clone();// clone to a different buffer
 385                         timestampWriteLock.unlock();
 386
 387                         System.out.println("Motion Detected (with percentage: " + Double.toString(percentMotion) + "%)");
 388                 }
 389
 390                 // Do output to the screen if we are using gui mode
 391                 if (DO_GRAPHICAL_USER_INTERFACE) {
 392
 393                         // change image data unto correct type for rendering
 394                         BufferedImage visualized1 = new BufferedImage(segmented.width, segmented.height, BufferedImage.TYPE_INT_RGB);
 395                         VisualizeBinaryData.renderBinary(segmented, false, visualized1);
 396
 397                         // change image data unto correct type for rendering
 398                         BufferedImage visualized2 = null;
 399                         visualized2 = ConvertBufferedImage.convertTo(newFrameFloat, visualized2, true);
 400
 401                         // place the images into the image grid
 402                         gui.setImage(0, 1, visualized1);
 403                         gui.setImage(0, 2, visualized2);
 404
 405                         // trigger rendering
 406                         gui.repaint();
 407                 }
 408         }
 409
 410         /** Method that constantly loops checking if the callbacks should be issues and
 411          *   issues the callbacks if they should be issues.
 412          *   This method should be run on a separate thread.
 413          *
 414          *   @return [void] None.
 415          */
 416         private void doCallbacks() {
 417
 418                 // Keep looping forever for callback
 419                 while (true) {
 420
 421                         // If motion detected
 422                         if (motionDetected.compareAndSet(true, false)) {
 423
 424                                 // Motion was detected so issue callbacks to all objects that registered
 425                                 // to receive callback from this class.
 426                                 for (MotionDetectionCallback c : callbackList) {
 427                                         //try {
 428                                                 c.motionDetected(this);
 429                                         //} catch (RemoteException re) {
 430                                         //}
 431                                 }
 432
 433                         } else {
 434
 435                                 // Sleep for 15 millisec to give time for new frame to arrive
 436                                 try {
 437                                         Thread.sleep(15);
 438                                 } catch (InterruptedException ie) {
 439                                 }
 440                         }
 441                 }
 442         }
 443
 444
 445
 446
 447         // /*******************************************************************************************************************************************
 448         // **  Main Method used for testing
 449         // *******************************************************************************************************************************************/
 450         // public static void main(String[] args) {
 451         //     MotionDetection mo = new MotionDetection(12, 0.5f, 10, 10);
 452
 453         //     AmcrestCamera cam = null;
 454         //     try {
 455
 456         //         InetAddress addr = InetAddress.getByName("192.168.1.29");
 457         //         cam = new AmcrestCamera(addr, "admin", "55779CatSoundz32");
 458         //         cam.registerCallback(mo);
 459         //         cam.start();
 460         //         // cam.streamDisconnect();
 461
 462         //     } catch (Exception e) {
 463
 464         //     }
 465
 466         //     while (true) {
 467
 468         //     }
 469
 470         // }
 471 }
 472