Adding feature to hold multiple hostnames and lists of packet orders in FlowPattern...

diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FlowPattern.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FlowPattern.java
index fb0433c8e1a7626eee0f99fcd5e41288185530da..aca63ab36f37e107d69795915d18cb5d436fc34c 100644 (file)
--- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FlowPattern.java
+++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FlowPattern.java
@@ -5,44 +5,24 @@ import org.pcap4j.packet.*;
 import org.pcap4j.packet.DnsPacket;
 import org.pcap4j.packet.namednumber.DnsResourceRecordType;
 
 import org.pcap4j.packet.DnsPacket;
 import org.pcap4j.packet.namednumber.DnsResourceRecordType;
 

-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-

 import java.io.EOFException;
 import java.io.EOFException;

+import java.net.InetAddress;

 import java.net.UnknownHostException;
 import java.net.UnknownHostException;

+import java.util.*;

 import java.util.concurrent.TimeoutException;
 
 /**
  * TODO add class documentation.
 import java.util.concurrent.TimeoutException;
 
 /**
  * TODO add class documentation.

+ * TODO: At this point, this class is still in transition to having multiple hostnames and lists of packets

  *
  * @author Janus Varmarken
  */
 public class FlowPattern {
 
  *
  * @author Janus Varmarken
  */
 public class FlowPattern {
 

-    static {
-        // TP-Link Local ON packet lengths (TCP payload only), extracted from ON event at Feb 13, 2018 13:38:04
-        // of the 5 switch data collection:
-        // 517 1448 1448 1448 855 191 51 490 1027 31
-
-        ArrayList<Integer> packetLengths = new ArrayList<>();
-        packetLengths.addAll(Arrays.asList(new Integer[] {517, 1448, 1448, 1448, 855, 191, 51, 490, 1027, 31}));
-        TP_LINK_LOCAL_ON = new FlowPattern("TP_LINK_LOCAL_ON", "events.tplinkra.com", packetLengths);
-    }
-
-    public static final FlowPattern TP_LINK_LOCAL_ON;
-

     /**
      * Class properties
      */
     /**
      * Class properties
      */

-    private final String patternId;
-
-    /**
-     * The hostname that this {@code FlowPattern} is associated with.
-     */
+    private final String mPatternId;

     private final String hostname;  // The hostname that this {@code FlowPattern} is associated with.
 
     /**
     private final String hostname;  // The hostname that this {@code FlowPattern} is associated with.
 
     /**


@@ -50,9 +30,10 @@ public class FlowPattern {
      * TODO: this is a simplified representation, we should also include information about direction of each packet.
      */
     private final List<Integer> flowPacketOrder;
      * TODO: this is a simplified representation, we should also include information about direction of each packet.
      */
     private final List<Integer> flowPacketOrder;

-        
-    private final Map<String, List<Integer>> hostnameToPacketOrderMap;
-    private final PcapHandle pcap;
+    private final Map<String, List<Integer>> mHostnameToPacketLengthsMap;
+    private final List<String> mHostnameList;
+    private final PcapHandle mPcap;
+

     
     /**
      * Class constants
     
     /**
      * Class constants


@@ -62,15 +43,17 @@ public class FlowPattern {
     /**
      * Constructor #1
      */
     /**
      * Constructor #1
      */

-    public FlowPattern(String patternId, String hostname, PcapHandle pcap) {
-        this.patternId = patternId;
+    public FlowPattern(String mPatternId, String hostname, PcapHandle mPcap) {
+        this.mPatternId = mPatternId;

         this.hostname = hostname;
         this.hostname = hostname;

-        this.pcap = pcap;
-        this.hostnameToPacketOrderMap = null;
+        this.mHostnameList = null;
+        this.mPcap = mPcap;
+        this.mHostnameToPacketLengthsMap = null;

         this.flowPacketOrder = new ArrayList<Integer>();
         processPcap();
     }
 
         this.flowPacketOrder = new ArrayList<Integer>();
         processPcap();
     }
 

+

     /**
      * Process the PcapHandle to strip off unnecessary packets and just get the integer array of packet lengths
      */
     /**
      * Process the PcapHandle to strip off unnecessary packets and just get the integer array of packet lengths
      */


@@ -78,7 +61,7 @@ public class FlowPattern {
 
         PcapPacket packet;
         try {
 
         PcapPacket packet;
         try {

-            while ((packet = pcap.getNextPacketEx()) != null) {
+            while ((packet = mPcap.getNextPacketEx()) != null) {

                 // For now, we only work support pattern search in TCP over IPv4.
                 IpV4Packet ipPacket = packet.get(IpV4Packet.class);
                 TcpPacket tcpPacket = packet.get(TcpPacket.class);
                 // For now, we only work support pattern search in TCP over IPv4.
                 IpV4Packet ipPacket = packet.get(IpV4Packet.class);
                 TcpPacket tcpPacket = packet.get(TcpPacket.class);


@@ -91,7 +74,7 @@ public class FlowPattern {
             }
         } catch (EOFException eofe) {
             System.out.println("[ FlowPattern ] Finished processing a training PCAP stream!");
             }
         } catch (EOFException eofe) {
             System.out.println("[ FlowPattern ] Finished processing a training PCAP stream!");

-            System.out.println("[ FlowPattern ] Pattern for " + patternId + ": " + Arrays.toString(flowPacketOrder.toArray()));
+            System.out.println("[ FlowPattern ] Pattern for " + mPatternId + ": " + Arrays.toString(flowPacketOrder.toArray()));

         } catch (PcapNativeException  |
                  TimeoutException     |
                  NotOpenException ex) {
         } catch (PcapNativeException  |
                  TimeoutException     |
                  NotOpenException ex) {


@@ -99,47 +82,119 @@ public class FlowPattern {
         }
     }
 
         }
     }
 

+

     /**
     /**

-     * Constructor #2
-     *
-     * @param   patternId       Label for this pattern
-     * @param   hostname        Hostname associated with this pattern
-     * @param   flowPacketOrder List of packets in order
+     * Process the PcapHandle to strip off unnecessary packets.
+     * We then map list of hostnames to their respective arrays of packet lengths

      */
      */

-    public FlowPattern(String patternId, String hostname, List<Integer> flowPacketOrder) {
-        this.patternId = patternId;
-        this.hostname = hostname;
-        this.hostnameToPacketOrderMap = null;
-        this.pcap = null;
-        this.flowPacketOrder = Collections.unmodifiableList(flowPacketOrder);
+    private void processPcapToMap() {
+
+        PcapPacket packet;
+        try {
+            int hostIndex = -1;
+            Set<String> addressSet = new HashSet<>();
+            while ((packet = mPcap.getNextPacketEx()) != null) {
+                // For now, we only work support pattern search in TCP over IPv4.
+                IpV4Packet ipPacket = packet.get(IpV4Packet.class);
+                TcpPacket tcpPacket = packet.get(TcpPacket.class);
+                if (ipPacket == null || tcpPacket == null) {
+                    continue;
+                }
+                if (tcpPacket.getPayload() == null) {
+                // We skip non-payload control packets as these are less predictable
+                    continue;
+                }
+                // We assume that if it is not a local address then it is a cloud server address
+                InetAddress srcAddress = ipPacket.getHeader().getSrcAddr();
+                InetAddress dstAddress = ipPacket.getHeader().getDstAddr();
+                boolean fromServer = !srcAddress.isSiteLocalAddress();
+                boolean fromClient = !dstAddress.isSiteLocalAddress();
+                if (!fromServer && !fromClient) {
+                    // Packet not related to pattern, skip it
+                    continue;
+                } else {
+                    // We relate and assume that this address is from our cloud server
+                    String cloudAddress = null;
+                    if (fromClient) {
+                        cloudAddress = dstAddress.getHostAddress();
+                    } else { // fromServer
+                        cloudAddress = srcAddress.getHostAddress();
+                    }
+                    //System.out.println("\nCloud address: " + cloudAddress);
+                    if (!addressSet.contains(cloudAddress)) {
+                        addressSet.add(cloudAddress);
+                        hostIndex++;
+                    }
+
+                    String hostname = mHostnameList.get(hostIndex);
+                    List<Integer> packetLengthsList = mHostnameToPacketLengthsMap.containsKey(hostname) ? 
+                        mHostnameToPacketLengthsMap.get(hostname) : new ArrayList<>();
+                    int packetLength = tcpPacket.getPayload().length();
+                    packetLengthsList.add(packetLength);
+                    mHostnameToPacketLengthsMap.put(hostname, packetLengthsList);
+                }
+            }
+        } catch (EOFException eofe) {
+            System.out.println("[ FlowPattern ] Finished processing a training PCAP stream!");
+            System.out.println("[ FlowPattern ] Pattern for " + mPatternId + ": " + Arrays.toString(mHostnameToPacketLengthsMap.entrySet().toArray()));
+        } catch (PcapNativeException  |
+                 TimeoutException     |
+                 NotOpenException ex) {
+            ex.printStackTrace();
+        }

     }
     }

+

     
     /**
     
     /**

-     * Constructor #3
+     * Constructor #2

      */
      */

-    public FlowPattern(String patternId, String hostname, Map<String, List<Integer>> hostnameToPacketOrderMap) {
-        this.patternId = patternId;
-        this.hostname = hostname;
-        this.pcap = null;
+    public FlowPattern(String mPatternId, List<String> mHostnameList, PcapHandle mPcap) {
+        this.mPatternId = mPatternId;
+        this.hostname = null;
+        this.mHostnameList = mHostnameList;
+        this.mPcap = mPcap;

         this.flowPacketOrder = null;
         this.flowPacketOrder = null;

-        this.hostnameToPacketOrderMap = Collections.unmodifiableMap(hostnameToPacketOrderMap);
+        this.mHostnameToPacketLengthsMap = new HashMap<>();
+        processPcapToMap();

     }
 
     }
 

+

     public String getPatternId() {
     public String getPatternId() {

-        return patternId;
+        return mPatternId;

     }
 
     }
 

+

     public String getHostname() {
         return hostname;
     }
 
     public String getHostname() {
         return hostname;
     }
 

+

     /**
     /**

-     * Get the the sequence of packet lengths that defines this {@code FlowPattern}.
+     * Get the sequence of packet lengths that defines this {@code FlowPattern}.

      * @return the sequence of packet lengths that defines this {@code FlowPattern}.
      */
     public List<Integer> getPacketOrder() {
         return flowPacketOrder;
     }
      * @return the sequence of packet lengths that defines this {@code FlowPattern}.
      */
     public List<Integer> getPacketOrder() {
         return flowPacketOrder;
     }

+
+
+    /**
+     * Get the sequence of packet lengths based on input hostname.
+     * @return the sequence of packet lengths that defines this {@code FlowPattern}.
+     */
+    public List<Integer> getPacketOrder(String hostname) {
+        return mHostnameToPacketLengthsMap.get(hostname);
+    }
+
+
+    /**
+     * Get the list of associated hostnames.
+     * @return the associated hostnames that define this {@code FlowPattern}.
+     */
+    public List<String> getHostnameList() {
+        return mHostnameList;
+    }
+

     
     /**
      * Get the length of the List of {@code FlowPattern}.
     
     /**
      * Get the length of the List of {@code FlowPattern}.


@@ -147,6 +202,14 @@ public class FlowPattern {
      */
     public int getLength() {
         return flowPacketOrder.size();
      */
     public int getLength() {
         return flowPacketOrder.size();

-    }
+    }  
+

 
 

+    /**
+     * Get the length of the List of {@code FlowPattern}.
+     * @return the length of the List of {@code FlowPattern}.
+     */
+    public int getLength(String hostname) {
+        return mHostnameToPacketLengthsMap.get(hostname).size();
+    } 

 }
 }

diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FlowPatternFinder.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FlowPatternFinder.java
index 115294645a8820df22a7531b83b8ffc12b741a3e..da0d3eb53f4e57a55825b7a6b4fc27f421367760 100644 (file)
--- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FlowPatternFinder.java
+++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FlowPatternFinder.java
@@ -90,6 +90,10 @@ public class FlowPatternFinder {
     private void findFlowPattern() {
         try {
             PcapPacket packet;
     private void findFlowPattern() {
         try {
             PcapPacket packet;

+//            TODO: The new comparison method is pending
+//            TODO: For now, just compare using one hostname and one list per FlowPattern
+//            List<String> hostnameList = mPattern.getHostnameList();
+//            int hostIndex = 0;

             int patternLength = mPattern.getLength();
             while ((packet = mPcap.getNextPacketEx()) != null) {
                 // Let DnsMap handle DNS packets.
             int patternLength = mPattern.getLength();
             while ((packet = mPcap.getNextPacketEx()) != null) {
                 // Let DnsMap handle DNS packets.


@@ -104,6 +108,10 @@ public class FlowPatternFinder {
                 if (ipPacket == null || tcpPacket == null) {
                     continue;
                 }
                 if (ipPacket == null || tcpPacket == null) {
                     continue;
                 }

+                if (tcpPacket.getPayload() == null) {
+                    // We skip non-payload control packets as these are less predictable
+                    continue;
+                }

                 String srcAddress = ipPacket.getHeader().getSrcAddr().getHostAddress();
                 String dstAddress = ipPacket.getHeader().getDstAddr().getHostAddress();
                 int srcPort = tcpPacket.getHeader().getSrcPort().valueAsInt();
                 String srcAddress = ipPacket.getHeader().getSrcAddr().getHostAddress();
                 String dstAddress = ipPacket.getHeader().getDstAddr().getHostAddress();
                 int srcPort = tcpPacket.getHeader().getSrcPort().valueAsInt();


@@ -111,14 +119,13 @@ public class FlowPatternFinder {
                 // Is this packet related to the pattern; i.e. is it going to (or coming from) the cloud server?
                 boolean fromServer = mDnsMap.isRelatedToCloudServer(srcAddress, mPattern.getHostname());
                 boolean fromClient = mDnsMap.isRelatedToCloudServer(dstAddress, mPattern.getHostname());
                 // Is this packet related to the pattern; i.e. is it going to (or coming from) the cloud server?
                 boolean fromServer = mDnsMap.isRelatedToCloudServer(srcAddress, mPattern.getHostname());
                 boolean fromClient = mDnsMap.isRelatedToCloudServer(dstAddress, mPattern.getHostname());

+//                String currentHostname = hostnameList.get(hostIndex);
+//                boolean fromServer = mDnsMap.isRelatedToCloudServer(srcAddress, currentHostname);
+//                boolean fromClient = mDnsMap.isRelatedToCloudServer(dstAddress, currentHostname);

                 if (!fromServer && !fromClient) {
                     // Packet not related to pattern, skip it.
                     continue;
                 }
                 if (!fromServer && !fromClient) {
                     // Packet not related to pattern, skip it.
                     continue;
                 }

-                if (tcpPacket.getPayload() == null) {
-                    // We skip non-payload control packets as these are less predictable
-                    continue;
-                }

                 // Conversations (connections/sessions) are identified by the four-tuple
                 // (clientIp, clientPort, serverIp, serverPort) (see Conversation Javadoc).
                 // Create "dummy" conversation for looking up an existing entry.
                 // Conversations (connections/sessions) are identified by the four-tuple
                 // (clientIp, clientPort, serverIp, serverPort) (see Conversation Javadoc).
                 // Create "dummy" conversation for looking up an existing entry.


@@ -138,6 +145,7 @@ public class FlowPatternFinder {
                 // Refresh reference to point to entry in map (in case packet was added to existing entry).
                 conversation = mConversations.get(conversation);
                 if (conversation.getPackets().size() == mPattern.getLength()) {
                 // Refresh reference to point to entry in map (in case packet was added to existing entry).
                 conversation = mConversations.get(conversation);
                 if (conversation.getPackets().size() == mPattern.getLength()) {

+//                if (conversation.getPackets().size() == mPattern.getLength(currentHostname)) {

                     // Conversation reached a size that matches the expected size.
                     // Remove the Conversation from the map and start the analysis.
                     // Any future packets identified by the same four tuple will be tied to a new Conversation instance.
                     // Conversation reached a size that matches the expected size.
                     // Remove the Conversation from the map and start the analysis.
                     // Any future packets identified by the same four tuple will be tied to a new Conversation instance.


@@ -147,6 +155,8 @@ public class FlowPatternFinder {
                     PatternComparisonTask<CompleteMatchPatternComparisonResult> comparisonTask =
                             new PatternComparisonTask<>(conversation, mPattern, ComparisonFunctions.COMPLETE_MATCH);
                     mPendingComparisons.add(EXECUTOR_SERVICE.submit(comparisonTask));
                     PatternComparisonTask<CompleteMatchPatternComparisonResult> comparisonTask =
                             new PatternComparisonTask<>(conversation, mPattern, ComparisonFunctions.COMPLETE_MATCH);
                     mPendingComparisons.add(EXECUTOR_SERVICE.submit(comparisonTask));

+                    // Increment hostIndex to find the next
+                    

                 }
             }
         } catch (EOFException eofe) {
                 }
             }
         } catch (EOFException eofe) {

diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java
index 239d8c18fdffb363f153c017d8703361056eca64..b828323f5a0c5268d53d93bcb6adf7c9e3309743 100644 (file)
--- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java
+++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java
@@ -4,6 +4,7 @@ import org.pcap4j.core.*;
 
 import java.io.EOFException;
 import java.net.UnknownHostException;
 
 import java.io.EOFException;
 import java.net.UnknownHostException;

+import java.util.*;

 import java.util.concurrent.TimeoutException;
 
 /**
 import java.util.concurrent.TimeoutException;
 
 /**


@@ -21,7 +22,8 @@ public class Main {
 
     public static void main(String[] args) throws PcapNativeException, NotOpenException, EOFException, TimeoutException, UnknownHostException {
         final String fileName = args.length > 0 ? args[0] : "/home/rtrimana/pcap_processing/smart_home_traffic/Code/Projects/SmartPlugDetector/pcap/wlan1.local.remote.dns.pcap";
 
     public static void main(String[] args) throws PcapNativeException, NotOpenException, EOFException, TimeoutException, UnknownHostException {
         final String fileName = args.length > 0 ? args[0] : "/home/rtrimana/pcap_processing/smart_home_traffic/Code/Projects/SmartPlugDetector/pcap/wlan1.local.remote.dns.pcap";

-        final String trainingFileName = "./pcap/TP_LINK_LOCAL_OFF.pcap";
+        final String trainingFileName = "./pcap/TP_LINK_LOCAL_ON.pcap";
+        //final String trainingFileName = "./pcap/TP_LINK_REMOTE_ON.pcap";

 
         // ====== Debug code ======
         PcapHandle handle;
 
         // ====== Debug code ======
         PcapHandle handle;


@@ -33,9 +35,17 @@ public class Main {
             handle = Pcaps.openOffline(fileName);
             trainingPcap = Pcaps.openOffline(trainingFileName);
         }
             handle = Pcaps.openOffline(fileName);
             trainingPcap = Pcaps.openOffline(trainingFileName);
         }

-        FlowPattern fp = new FlowPattern("TP_LINK_LOCAL_OFF", "events.tplinkra.com", trainingPcap);
-        
-        //FlowPatternFinder fpf = new FlowPatternFinder(handle, FlowPattern.TP_LINK_LOCAL_ON);
+
+        // TODO: The followings are the way to extract multiple hostnames and their associated packet lengths lists
+        //List<String> list = new ArrayList<>();
+        //list.add("events.tplinkra.com");
+        //FlowPattern fp = new FlowPattern("TP_LINK_LOCAL_ON", list, trainingPcap);
+        //List<String> list2 = new ArrayList<>();
+        //list2.add("devs.tplinkcloud.com");
+        //list2.add("events.tplinkra.com");
+        //FlowPattern fp3 = new FlowPattern("TP_LINK_REMOTE_ON", list2, trainingPcap);
+
+        FlowPattern fp = new FlowPattern("TP_LINK_LOCAL_ON", "events.tplinkra.com", trainingPcap);        

         FlowPatternFinder fpf = new FlowPatternFinder(handle, fp);
         fpf.start();
 
         FlowPatternFinder fpf = new FlowPatternFinder(handle, fp);
         fpf.start();
 

diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/PatternComparisonTask.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/PatternComparisonTask.java
index 705646c0e300358e59c543f22d8156f9b245ecdc..808dee73fab6647e84f7139e734a91a382332cc3 100644 (file)
--- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/PatternComparisonTask.java
+++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/PatternComparisonTask.java
@@ -17,7 +17,7 @@ public class PatternComparisonTask<R extends AbstractPatternComparisonResult<?>>
 
     private final Conversation mConversation;
     private final FlowPattern mFlowPattern;
 
     private final Conversation mConversation;
     private final FlowPattern mFlowPattern;

-    private final BiFunction<Conversation, FlowPattern, R> mComparitor;
+    private final BiFunction<Conversation, FlowPattern, R> mComparator;

 
     /**
      * Create a new {@code PatternComparisonTask}.
 
     /**
      * Create a new {@code PatternComparisonTask}.


@@ -29,12 +29,12 @@ public class PatternComparisonTask<R extends AbstractPatternComparisonResult<?>>
     public PatternComparisonTask(Conversation conversation, FlowPattern pattern, BiFunction<Conversation, FlowPattern, R> comparisonFunction) {
         this.mConversation = conversation;
         this.mFlowPattern = pattern;
     public PatternComparisonTask(Conversation conversation, FlowPattern pattern, BiFunction<Conversation, FlowPattern, R> comparisonFunction) {
         this.mConversation = conversation;
         this.mFlowPattern = pattern;

-        this.mComparitor = comparisonFunction;
+        this.mComparator = comparisonFunction;

     }
 
     @Override
     public R call() throws Exception {
     }
 
     @Override
     public R call() throws Exception {

-        return mComparitor.apply(mConversation, mFlowPattern);
+        return mComparator.apply(mConversation, mFlowPattern);

     }
 
     }
 

-}
\ No newline at end of file
+}