Implementing relaxed matching for layer 2 and layer 3.
[pingpong.git] / Code / Projects / PacketLevelSignatureExtractor / src / main / java / edu / uci / iotproject / detection / layer2 / Layer2SignatureDetector.java
index c133b8e..1984e7b 100644 (file)
@@ -39,6 +39,17 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb
      */
     private static boolean DUPLICATE_OUTPUT_TO_STD_OUT = true;
 
+    /**
+     * Router's MAC.
+     * This is only useful for the filter for direction when it is a WAN signature (Phone-Cloud or Device-Cloud).
+     * Phone-Device signatures do not have router MAC address in it.
+     */
+    // TODO: We can remove the following constants if we do remove router's MAC filtering for directions
+    private static String TRAINING_ROUTER_WLAN_MAC = null;
+    private static String ROUTER_WLAN_MAC = null;
+    //private static String TRAINING_ROUTER_WLAN_MAC = "b0:b9:8a:73:69:8e";
+    //private static String ROUTER_WLAN_MAC = "00:c1:b1:14:eb:31";
+
     private static List<Function<Layer2Flow, Boolean>> parseSignatureMacFilters(String filtersString) {
         List<Function<Layer2Flow, Boolean>> filters = new ArrayList<>();
         String[] filterRegexes = filtersString.split(";");
@@ -51,51 +62,77 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb
     }
 
     public static void main(String[] args) throws PcapNativeException, NotOpenException, IOException {
+        String errMsg = String.format("SPECTO version 1.0\n" +
+                        "Copyright (C) 2018-2019 Janus Varmarken and Rahmadi Trimananda.\n" +
+                        "University of California, Irvine.\n" +
+                        "All rights reserved.\n\n" +
+                        "Usage: %s inputPcapFile onAnalysisFile offAnalysisFile onSignatureFile offSignatureFile " +
+                        "resultsFile signatureDuration eps onMaxSkippedPackets offMaxSkippedPackets" +
+                        "\n  inputPcapFile: the target of the detection" +
+                        "\n  onAnalysisFile: the file that contains the ON clusters analysis" +
+                        "\n  offAnalysisFile: the file that contains the OFF clusters analysis" +
+                        "\n  onSignatureFile: the file that contains the ON signature to search for" +
+                        "\n  offSignatureFile: the file that contains the OFF signature to search for" +
+                        "\n  resultsFile: where to write the results of the detection" +
+                        "\n  signatureDuration: the maximum duration of signature detection" +
+                        "\n  epsilon: the epsilon value for the DBSCAN algorithm\n" +
+                        "\n  Additional options (add '-r' before the following two parameters):" +
+                        "\n  delta: delta for relaxed matching" +
+                        "\n  packetId: packet number in the sequence" +
+                        "\n            (could be more than one packet whose matching is relaxed, " +
+                        "\n             e.g., 0,1 for packets 0 and 1)",
+                Layer2SignatureDetector.class.getSimpleName());
+        String optParamsExplained = "Above are the required, positional arguments. In addition to these, the " +
+                "following options and associated positional arguments may be used:\n" +
+                "  '-onmacfilters <regex>;<regex>;...;<regex>' which specifies that sequence matching should ONLY" +
+                " be performed on flows where the MAC of one of the two endpoints matches the given regex. Note " +
+                "that you MUST specify a regex for each cluster of the signature. This is to facilitate more " +
+                "aggressive filtering on parts of the signature (e.g., the communication that involves the " +
+                "smart home device itself as one can drop all flows that do not include an endpoint with a MAC " +
+                "that matches the vendor's prefix).\n" +
+                "  '-offmacfilters <regex>;<regex>;...;<regex>' works exactly the same as onmacfilters, but " +
+                "applies to the OFF signature instead of the ON signature.\n" +
+                "  '-sout <boolean literal>' true/false literal indicating if output should also be printed to std out; default is true.\n" +
+                "  '-vpn <router mac>' router's MAC address; this is to simulate a VPN that combines all flows even when the traffic is not a VPN traffic.\n" +
+                "  '-onskipped <max duration of on-signature>' the maximum duration of ON signature detection.\n" +
+                "  '-offskipped <max duration of off-signature>' the maximum duration of OFF signature detection.\n";
         // Parse required parameters.
-//        if (args.length < 7) {
-        if (args.length < 5) {
-            String errMsg = String.format("Usage: %s inputPcapFile onAnalysisFile offAnalysisFile onSignatureFile offSignatureFile resultsFile" +
-                            "\n  inputPcapFile: the target of the detection" +
-//                            "\n  onAnalysisFile: the file that contains the ON clusters analysis" +
-//                            "\n  offAnalysisFile: the file that contains the OFF clusters analysis" +
-                            "\n  onSignatureFile: the file that contains the ON signature to search for" +
-                            "\n  offSignatureFile: the file that contains the OFF signature to search for" +
-                            "\n  resultsFile: where to write the results of the detection" +
-                            "\n  signatureDuration: the maximum duration of signature detection",
-                    Layer2SignatureDetector.class.getSimpleName());
+        if (args.length < 8) {
             System.out.println(errMsg);
-            String optParamsExplained = "Above are the required, positional arguments. In addition to these, the " +
-                    "following options and associated positional arguments may be used:\n" +
-                    "  '-onmacfilters <regex>;<regex>;...;<regex>' which specifies that sequence matching should ONLY" +
-                    " be performed on flows where the MAC of one of the two endpoints matches the given regex. Note " +
-                    "that you MUST specify a regex for each cluster of the signature. This is to facilitate more " +
-                    "aggressive filtering on parts of the signature (e.g., the communication that involves the " +
-                    "smart home device itself as one can drop all flows that do not include an endpoint with a MAC " +
-                    "that matches the vendor's prefix).\n" +
-                    "  '-offmacfilters <regex>;<regex>;...;<regex>' works exactly the same as onmacfilters, but " +
-                    "applies to the OFF signature instead of the ON signature.\n" +
-                    "  '-sout <boolean literal>' true/false literal indicating if output should also be printed to std out; default is true.";
             System.out.println(optParamsExplained);
             return;
         }
-        // TODO: We could take 7 inputs if we decided to use the cluster analyses.
-//        final String pcapFile = args[0];
-//        final String onClusterAnalysisFile = args[1];
-//        final String offClusterAnalysisFile = args[2];
-//        final String onSignatureFile = args[3];
-//        final String offSignatureFile = args[4];
-//        final String resultsFile = args[5];
-//        final int signatureDuration = Integer.parseInt(args[6]);
-
         final String pcapFile = args[0];
-        final String onSignatureFile = args[1];
-        final String offSignatureFile = args[2];
-        final String resultsFile = args[3];
-        final int signatureDuration = Integer.parseInt(args[4]);
+        final String onClusterAnalysisFile = args[1];
+        final String offClusterAnalysisFile = args[2];
+        final String onSignatureFile = args[3];
+        final String offSignatureFile = args[4];
+        final String resultsFile = args[5];
+        final int signatureDuration = Integer.parseInt(args[6]);
+        final double eps = Double.parseDouble(args[7]);
+        // Additional feature---relaxed matching
+        int delta = 0;
+        final Set<Integer> packetSet = new HashSet<>();
+        if (args.length > 8 && args[8].equals("-r")) {
+            delta = Integer.parseInt(args[9]);
+            StringTokenizer stringTokenizerOff = new StringTokenizer(args[10], ",");
+            // Add the list of packet IDs
+            while(stringTokenizerOff.hasMoreTokens()) {
+                int id = Integer.parseInt(stringTokenizerOff.nextToken());
+                packetSet.add(id);
+            }
+        } else {
+            System.out.println(errMsg);
+            System.out.println(optParamsExplained);
+            return;
+        }
 
         // Parse optional parameters.
         List<Function<Layer2Flow, Boolean>> onSignatureMacFilters = null, offSignatureMacFilters = null;
-        final int optParamsStartIdx = 7;
+        String vpnClientMacAddress = null;
+        int onMaxSkippedPackets = -1;
+        int offMaxSkippedPackets = -1;
+        final int optParamsStartIdx = 8;
         if (args.length > optParamsStartIdx) {
             for (int i = optParamsStartIdx; i < args.length; i++) {
                 if (args[i].equalsIgnoreCase("-onMacFilters")) {
@@ -107,6 +144,16 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb
                 } else if (args[i].equalsIgnoreCase("-sout")) {
                     // Next argument is a boolean true/false literal.
                     DUPLICATE_OUTPUT_TO_STD_OUT = Boolean.parseBoolean(args[i+1]);
+                } else if (args[i].equalsIgnoreCase("-vpn")) {
+                    vpnClientMacAddress = args[i+1];
+                } else if (args[i].equalsIgnoreCase("-onskipped")) {
+                    if (i+2 > args.length - 1 || !args[i+2].equalsIgnoreCase("-offskipped")) {
+                        throw new Error("Please make sure that the -onskipped and -offskipped options are both used at the same time...");
+                    }
+                    if (args[i+2].equalsIgnoreCase("-offskipped")) {
+                        onMaxSkippedPackets = Integer.parseInt(args[i+1]);
+                        offMaxSkippedPackets = Integer.parseInt(args[i+3]);
+                    }
                 }
             }
         }
@@ -118,47 +165,56 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb
         // Include metadata as comments at the top
         PrintWriterUtils.println("# Detection results for:", resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
         PrintWriterUtils.println("# - inputPcapFile: " + pcapFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
-//        PrintWriterUtils.println("# - onAnalysisFile: " + onClusterAnalysisFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
-//        PrintWriterUtils.println("# - offAnalysisFile: " + offClusterAnalysisFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
+        PrintWriterUtils.println("# - onAnalysisFile: " + onClusterAnalysisFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
+        PrintWriterUtils.println("# - offAnalysisFile: " + offClusterAnalysisFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
         PrintWriterUtils.println("# - onSignatureFile: " + onSignatureFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
         PrintWriterUtils.println("# - offSignatureFile: " + offSignatureFile, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
         resultsWriter.flush();
 
-        double eps = 10.0;
         // Create signature detectors and add observers that output their detected events.
         List<List<List<PcapPacket>>> onSignature = PrintUtils.deserializeFromFile(onSignatureFile);
         List<List<List<PcapPacket>>> offSignature = PrintUtils.deserializeFromFile(offSignatureFile);
         // Load signature analyses
-//        List<List<List<PcapPacket>>> onClusterAnalysis = PrintUtils.deserializeFromFile(onClusterAnalysisFile);
-//        List<List<List<PcapPacket>>> offClusterAnalysis = PrintUtils.deserializeFromFile(offClusterAnalysisFile);
+        List<List<List<PcapPacket>>> onClusterAnalysis = PrintUtils.deserializeFromFile(onClusterAnalysisFile);
+        List<List<List<PcapPacket>>> offClusterAnalysis = PrintUtils.deserializeFromFile(offClusterAnalysisFile);
         // TODO: FOR NOW WE DECIDE PER SIGNATURE AND THEN WE OR THE BOOLEANS
         // TODO: SINCE WE ONLY HAVE 2 SIGNATURES FOR NOW (ON AND OFF), THEN IT IS USUALLY EITHER RANGE-BASED OR
         // TODO: STRICT MATCHING
         // Check if we should use range-based matching
-//        boolean isRangeBasedForOn = PcapPacketUtils.isRangeBasedMatching(onSignature, eps, offSignature);
-//        boolean isRangeBasedForOff = PcapPacketUtils.isRangeBasedMatching(offSignature, eps, onSignature);
-//        // Update the signature with ranges if it is range-based
-//        if (isRangeBasedForOn && isRangeBasedForOff) {
-//            onSignature = PcapPacketUtils.useRangeBasedMatching(onSignature, onClusterAnalysis);
-//            offSignature = PcapPacketUtils.useRangeBasedMatching(offSignature, offClusterAnalysis);
-//        }
+        boolean isRangeBasedForOn = PcapPacketUtils.isRangeBasedMatching(onSignature, eps, offSignature);
+        boolean isRangeBasedForOff = PcapPacketUtils.isRangeBasedMatching(offSignature, eps, onSignature);
         // TODO: WE DON'T DO RANGE-BASED FOR NOW BECAUSE THE RESULTS ARE TERRIBLE FOR LAYER 2 MATCHING
         // TODO: THIS WOULD ONLY WORK FOR SIGNATURES LONGER THAN 2 PACKETS
-        boolean isRangeBasedForOn = false;
-        boolean isRangeBasedForOff = false;
+//        boolean isRangeBasedForOn = false;
+//        boolean isRangeBasedForOff = false;
+        // Update the signature with ranges if it is range-based
+        if (isRangeBasedForOn) {
+            onSignature = PcapPacketUtils.useRangeBasedMatching(onSignature, onClusterAnalysis);
+        }
+        if (isRangeBasedForOff) {
+            offSignature = PcapPacketUtils.useRangeBasedMatching(offSignature, offClusterAnalysis);
+        }
         Layer2SignatureDetector onDetector = onSignatureMacFilters == null ?
-                new Layer2SignatureDetector(onSignature, isRangeBasedForOn, eps) :
-                new Layer2SignatureDetector(onSignature, onSignatureMacFilters, signatureDuration, isRangeBasedForOn, eps);
+                new Layer2SignatureDetector(onSignature, TRAINING_ROUTER_WLAN_MAC, ROUTER_WLAN_MAC, signatureDuration,
+                        isRangeBasedForOn, eps, onMaxSkippedPackets, vpnClientMacAddress, delta, packetSet) :
+                new Layer2SignatureDetector(onSignature, TRAINING_ROUTER_WLAN_MAC, ROUTER_WLAN_MAC,
+                        onSignatureMacFilters, signatureDuration, isRangeBasedForOn, eps, onMaxSkippedPackets,
+                        vpnClientMacAddress, delta, packetSet);
         Layer2SignatureDetector offDetector = offSignatureMacFilters == null ?
-                new Layer2SignatureDetector(offSignature, isRangeBasedForOff, eps) :
-                new Layer2SignatureDetector(offSignature, offSignatureMacFilters, signatureDuration, isRangeBasedForOff, eps);
+                new Layer2SignatureDetector(offSignature, TRAINING_ROUTER_WLAN_MAC, ROUTER_WLAN_MAC, signatureDuration,
+                        isRangeBasedForOff, eps, offMaxSkippedPackets, vpnClientMacAddress, delta, packetSet) :
+                new Layer2SignatureDetector(offSignature, TRAINING_ROUTER_WLAN_MAC, ROUTER_WLAN_MAC, offSignatureMacFilters,
+                        signatureDuration, isRangeBasedForOff, eps, offMaxSkippedPackets, vpnClientMacAddress, delta, packetSet);
+        final List<UserAction> detectedEvents = new ArrayList<>();
         onDetector.addObserver((signature, match) -> {
             UserAction event = new UserAction(UserAction.Type.TOGGLE_ON, match.get(0).get(0).getTimestamp());
             PrintWriterUtils.println(event, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
+            detectedEvents.add(event);
         });
         offDetector.addObserver((signature, match) -> {
             UserAction event = new UserAction(UserAction.Type.TOGGLE_OFF, match.get(0).get(0).getTimestamp());
             PrintWriterUtils.println(event, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
+            detectedEvents.add(event);
         });
 
         // Load the PCAP file
@@ -172,6 +228,27 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb
         // Parse the file
         reader.readFromHandle();
 
+        String resultOn = "# Number of detected events of type " + UserAction.Type.TOGGLE_ON + ": " +
+                detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_ON).count();
+        String resultOff = "# Number of detected events of type " + UserAction.Type.TOGGLE_OFF + ": " +
+                detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_OFF).count();
+        String onMaximumSkippedPackets = "# Maximum number of skipped packets in ON signature " +
+                Integer.toString(onDetector.getMaxSkippedPackets());
+        String offMaximumSkippedPackets = "# Maximum number of skipped packets in OFF signature " +
+                Integer.toString(offDetector.getMaxSkippedPackets());
+        PrintWriterUtils.println(resultOn, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
+        PrintWriterUtils.println(resultOff, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
+        // Perform the skipped packet analysis if needed
+        if (onMaxSkippedPackets != -1 && offMaxSkippedPackets != -1) {
+            PrintWriterUtils.println(onMaximumSkippedPackets, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
+            for (Integer skippedPackets : onDetector.getSkippedPackets()) {
+                PrintWriterUtils.println(skippedPackets, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
+            }
+            PrintWriterUtils.println(offMaximumSkippedPackets, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
+            for (Integer skippedPackets : offDetector.getSkippedPackets()) {
+                PrintWriterUtils.println(skippedPackets, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT);
+            }
+        }
         // Flush output to results file and close it.
         resultsWriter.flush();
         resultsWriter.close();
@@ -204,18 +281,31 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb
     /**
      * In charge of reassembling layer 2 packet flows.
      */
-    private final Layer2FlowReassembler mFlowReassembler = new Layer2FlowReassembler();
+    private Layer2FlowReassembler mFlowReassembler;
 
     private final List<SignatureDetectorObserver> mObservers = new ArrayList<>();
 
     private int mInclusionTimeMillis;
 
-    public Layer2SignatureDetector(List<List<List<PcapPacket>>> searchedSignature, boolean isRangeBased, double eps) {
-        this(searchedSignature, null, 0, isRangeBased, eps);
+    /**
+     * Skipped-packet analysis.
+     */
+    private int mMaxSkippedPackets;
+    private List<Integer> mSkippedPackets;
+
+
+
+    public Layer2SignatureDetector(List<List<List<PcapPacket>>> searchedSignature, String trainingRouterWlanMac,
+                                   String routerWlanMac, int signatureDuration, boolean isRangeBased, double eps,
+                                   int limitSkippedPackets, String vpnClientMacAddress, int delta, Set<Integer> packetSet) {
+        this(searchedSignature, trainingRouterWlanMac, routerWlanMac, null, signatureDuration, isRangeBased,
+                eps, limitSkippedPackets, vpnClientMacAddress, delta, packetSet);
     }
 
-    public Layer2SignatureDetector(List<List<List<PcapPacket>>> searchedSignature, List<Function<Layer2Flow,
-            Boolean>> flowFilters, int inclusionTimeMillis, boolean isRangeBased, double eps) {
+    public Layer2SignatureDetector(List<List<List<PcapPacket>>> searchedSignature, String trainingRouterWlanMac,
+                                   String routerWlanMac, List<Function<Layer2Flow, Boolean>> flowFilters,
+                                   int inclusionTimeMillis, boolean isRangeBased, double eps, int limitSkippedPackets,
+                                   String vpnClientMacAddress, int delta, Set<Integer> packetSet) {
         if (flowFilters != null && flowFilters.size() != searchedSignature.size()) {
             throw new IllegalArgumentException("If flow filters are used, there must be a flow filter for each cluster " +
                     "of the signature.");
@@ -225,8 +315,10 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb
         for (int i = 0; i < mSignature.size(); i++) {
             List<List<PcapPacket>> cluster = mSignature.get(i);
             Layer2ClusterMatcher clusterMatcher = flowFilters == null ?
-                    new Layer2ClusterMatcher(cluster, isRangeBased, eps) :
-                    new Layer2ClusterMatcher(cluster, flowFilters.get(i), isRangeBased, eps);
+                    new Layer2ClusterMatcher(cluster, trainingRouterWlanMac, routerWlanMac, inclusionTimeMillis,
+                            isRangeBased, eps, limitSkippedPackets, delta, packetSet) :
+                    new Layer2ClusterMatcher(cluster, trainingRouterWlanMac, routerWlanMac, flowFilters.get(i),
+                            inclusionTimeMillis, isRangeBased, eps, limitSkippedPackets, delta, packetSet);
             clusterMatcher.addObserver(this);
             clusterMatchers.add(clusterMatcher);
         }
@@ -241,9 +333,27 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb
         }
         mClusterMatcherIds = Collections.unmodifiableMap(clusterMatcherIds);
         // Register all cluster matchers to receive a notification whenever a new flow is encountered.
+        if (vpnClientMacAddress != null) {
+            mFlowReassembler = new Layer2FlowReassembler(vpnClientMacAddress);
+        } else {
+            mFlowReassembler = new Layer2FlowReassembler();
+        }
         mClusterMatchers.forEach(cm -> mFlowReassembler.addObserver(cm));
         mInclusionTimeMillis =
                 inclusionTimeMillis == 0 ? TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS : inclusionTimeMillis;
+        mMaxSkippedPackets = 0;
+        mSkippedPackets = new ArrayList<>();
+    }
+
+    public int getMaxSkippedPackets() {
+        return mMaxSkippedPackets;
+    }
+
+    public List<Integer> getSkippedPackets() {
+        for (Layer2ClusterMatcher matcher : mClusterMatchers) {
+            mSkippedPackets.addAll(matcher.getSkippedPackets());
+        }
+        return mSkippedPackets;
     }
 
     @Override
@@ -259,6 +369,10 @@ public class Layer2SignatureDetector implements PacketListener, ClusterMatcherOb
             // Add the match at the corresponding index
             mPendingMatches[mClusterMatcherIds.get(clusterMatcher)].add(match);
             checkSignatureMatch();
+            // Update maximum number of skipped packets
+            if (mMaxSkippedPackets < ((Layer2ClusterMatcher) clusterMatcher).getMaxSkippedPackets()) {
+                mMaxSkippedPackets = ((Layer2ClusterMatcher) clusterMatcher).getMaxSkippedPackets();
+            }
         }
     }