Fixing a bug: TcpReassembler needs to use the right router WAN's IP that should be...

diff --git a/Code/Projects/PacketLevelSignatureExtractor/execute_layer3_smarthome_all_detection_results_analysis.sh b/Code/Projects/PacketLevelSignatureExtractor/execute_layer3_smarthome_all_detection_results_analysis.sh
index 7eeba926e71112f0b6dcfd579162cefef93ac6f5..7e6360ce0ff5a7a1b6f7cb5b0e95d8f5932a933f 100755 (executable)
--- a/Code/Projects/PacketLevelSignatureExtractor/execute_layer3_smarthome_all_detection_results_analysis.sh
+++ b/Code/Projects/PacketLevelSignatureExtractor/execute_layer3_smarthome_all_detection_results_analysis.sh
@@ -484,7 +484,7 @@ RESULTS_FILE="$RESULTS_BASE_DIR/wemo-insight-plug/wemo-insight-plug.eth0.detecti
 ANALYSIS_RESULTS_FILE="$RESULTS_FILE.analysis"
 EXACT_MATCH="true"
 PROGRAM_ARGS="'$TIMESTAMPS_FILE' '$RESULTS_FILE' '$ANALYSIS_RESULTS_FILE' '$EXACT_MATCH'"
-./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
+#./gradlew run -DmainClass=edu.uci.iotproject.evaluation.DetectionResultsAnalyzer --args="$PROGRAM_ARGS"
 # ======================================================================================================================
 
 # ===================================================== WEMO PLUG ======================================================

diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/SignatureGenerator.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/SignatureGenerator.java
index bf456697dbb2079cf63312a855422ad6ea5fae08..15a79b78fc088b903eb45de445e23286498af302 100644 (file)
--- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/SignatureGenerator.java
+++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/SignatureGenerator.java
@@ -214,10 +214,10 @@ public class SignatureGenerator {
         // Perform clustering on conversation logged as part of all ON events.
         // Calculate number of events per type (only ON/only OFF), which means half of the number of all timestamps.
         int numberOfEventsPerType = triggerTimes.size() / 2;
-        int lowerBound = numberOfEventsPerType - (int)(numberOfEventsPerType * 0.1);
-        int upperBound = numberOfEventsPerType + (int)(numberOfEventsPerType * 0.1);
-        //int lowerBound = numberOfEventsPerType - (int)(numberOfEventsPerType * 0.8);
-        //int upperBound = numberOfEventsPerType + (int)(numberOfEventsPerType * 0.8);
+//        int lowerBound = numberOfEventsPerType - (int)(numberOfEventsPerType * 0.1);
+//        int upperBound = numberOfEventsPerType + (int)(numberOfEventsPerType * 0.1);
+        int lowerBound = numberOfEventsPerType - (int)(numberOfEventsPerType * 0.2);
+        int upperBound = numberOfEventsPerType + (int)(numberOfEventsPerType * 0.2);
         int minPts = lowerBound;
         DBSCANClusterer<PcapPacketPair> onClusterer = new DBSCANClusterer<>(eps, minPts);
         List<Cluster<PcapPacketPair>> onClusters = onClusterer.cluster(onPairs);

diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/TriggerTrafficExtractor.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/TriggerTrafficExtractor.java
index 0a22d63f09714e4323d2ea3a829a9cf0ed7a97d9..0adecd879affb0f5f6520eac4e574040890796e9 100644 (file)
--- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/TriggerTrafficExtractor.java
+++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/analysis/TriggerTrafficExtractor.java
@@ -27,6 +27,7 @@ public class TriggerTrafficExtractor implements PcapPacketFilter {
     private long mIncludedPackets = 0;
 
     public static final int INCLUSION_WINDOW_MILLIS = 15_000;
+    //public static final int INCLUSION_WINDOW_MILLIS = 30_000;
 
     public TriggerTrafficExtractor(String pcapFilePath, List<Instant> triggerTimes, String deviceIp) throws PcapNativeException, NotOpenException {
         mPcapFilePath = pcapFilePath;

diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3ClusterMatcher.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3ClusterMatcher.java
index 165cdb3e57f68de4319504a3fee4e15cd4efc34e..921fb634dbf46b9f68eae74ade69fbf336d10327 100644 (file)
--- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3ClusterMatcher.java
+++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/detection/layer3/Layer3ClusterMatcher.java
@@ -33,7 +33,7 @@ public class Layer3ClusterMatcher extends AbstractClusterMatcher implements Pack
     /**
      * For reassembling the observed traffic into TCP connections.
      */
-    private final TcpReassembler mTcpReassembler = new TcpReassembler();
+    private final TcpReassembler mTcpReassembler;
 
     /**
      * IP of the router's WAN port (if analyzed traffic is captured at the ISP's point of view).
@@ -91,6 +91,7 @@ public class Layer3ClusterMatcher extends AbstractClusterMatcher implements Pack
         }
         mEps = eps;
         mRouterWanIp = routerWanIp;
+                               mTcpReassembler = new TcpReassembler(mRouterWanIp);
         mInclusionTimeMillis =
                 inclusionTimeMillis == 0 ? TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS : inclusionTimeMillis;
     }
@@ -398,6 +399,138 @@ public class Layer3ClusterMatcher extends AbstractClusterMatcher implements Pack
         return Optional.empty();
     }
 
+    // TODO: EXPERIMENT WITH ONLY PACKET DIRECTION AND TIMING
+//    private Optional<List<PcapPacket>> findSubsequenceInSequence(List<PcapPacket> subsequence,
+//                                                                 List<PcapPacket> sequence,
+//                                                                 Conversation.Direction[] subsequenceDirections,
+//                                                                 Conversation.Direction[] sequenceDirections) {
+//        if (sequence.size() < subsequence.size()) {
+//            // If subsequence is longer, it cannot be contained in sequence.
+//            return Optional.empty();
+//        }
+//        if (isTlsSequence(subsequence) != isTlsSequence(sequence)) {
+//            // We consider it a mismatch if one is a TLS application data sequence and the other is not.
+//            return Optional.empty();
+//        }
+//        // If packet directions have not been precomputed by calling code, we need to construct them.
+//        if (subsequenceDirections == null) {
+//            subsequenceDirections = getPacketDirections(subsequence, mRouterWanIp);
+//        }
+//        if (sequenceDirections == null) {
+//            sequenceDirections = getPacketDirections(sequence, mRouterWanIp);
+//        }
+//        int subseqIdx = 0;
+//        int seqIdx = 0;
+//        while (subseqIdx < subsequence.size() && seqIdx < sequence.size()) {
+//            // We only have a match if packet lengths and directions match.
+//            if (subsequenceDirections[subseqIdx] == sequenceDirections[seqIdx]) {
+//                // A match; advance both indices to consider next packet in subsequence vs. next packet in sequence.
+//                subseqIdx++;
+//                seqIdx++;
+//                if (subseqIdx == subsequence.size()) {
+//                    // We managed to match the entire subsequence in sequence.
+//                    // Return the sublist of sequence that matches subsequence.
+//                    /*
+//                     * TODO:
+//                     * ASSUMES THE BACKING LIST (i.e., 'sequence') IS _NOT_ STRUCTURALLY MODIFIED, hence may not work
+//                     * for live traces!
+//                     */
+//                    // TODO: ALSO CHECK TIMING CONSTRAINT
+//                    PcapPacket firstPacket = sequence.get(seqIdx - subsequence.size());
+//                    PcapPacket lastPacket = sequence.get(seqIdx-1);
+//                    if (!lastPacket.getTimestamp().isAfter(firstPacket.getTimestamp().plusMillis(mInclusionTimeMillis))) {
+//                        return Optional.of(sequence.subList(seqIdx - subsequence.size(), seqIdx));
+//                    }
+//                }
+//            } else {
+//                // Mismatch.
+//                if (subseqIdx > 0) {
+//                    /*
+//                     * If we managed to match parts of subsequence, we restart the search for subsequence in sequence at
+//                     * the index of sequence where the current mismatch occurred. I.e., we must reset subseqIdx, but
+//                     * leave seqIdx untouched.
+//                     */
+//                    subseqIdx = 0;
+//                } else {
+//                    /*
+//                     * First packet of subsequence didn't match packet at seqIdx of sequence, so we move forward in
+//                     * sequence, i.e., we continue the search for subsequence in sequence starting at index seqIdx+1 of
+//                     * sequence.
+//                     */
+//                    seqIdx++;
+//                }
+//            }
+//        }
+//        return Optional.empty();
+//    }
+//
+//    private Optional<List<PcapPacket>> findSubsequenceInSequence(List<PcapPacket> lowerBound,
+//                                                                 List<PcapPacket> upperBound,
+//                                                                 List<PcapPacket> sequence,
+//                                                                 Conversation.Direction[] subsequenceDirections,
+//                                                                 Conversation.Direction[] sequenceDirections) {
+//        // Just do the checks for either lower or upper bound!
+//        // TODO: For now we use just the lower bound
+//        if (sequence.size() < lowerBound.size()) {
+//            // If subsequence is longer, it cannot be contained in sequence.
+//            return Optional.empty();
+//        }
+//        if (isTlsSequence(lowerBound) != isTlsSequence(sequence)) {
+//            // We consider it a mismatch if one is a TLS application data sequence and the other is not.
+//            return Optional.empty();
+//        }
+//        // If packet directions have not been precomputed by calling code, we need to construct them.
+//        if (subsequenceDirections == null) {
+//            subsequenceDirections = getPacketDirections(lowerBound, mRouterWanIp);
+//        }
+//        if (sequenceDirections == null) {
+//            sequenceDirections = getPacketDirections(sequence, mRouterWanIp);
+//        }
+//        int subseqIdx = 0;
+//        int seqIdx = 0;
+//        while (subseqIdx < lowerBound.size() && seqIdx < sequence.size()) {
+//            // TODO: ONLY MATCH PACKET DIRECTIONS
+//            if (subsequenceDirections[subseqIdx] == sequenceDirections[seqIdx]) {
+//                // A match; advance both indices to consider next packet in subsequence vs. next packet in sequence.
+//                subseqIdx++;
+//                seqIdx++;
+//                if (subseqIdx == lowerBound.size()) {
+//                    // We managed to match the entire subsequence in sequence.
+//                    // Return the sublist of sequence that matches subsequence.
+//                    /*
+//                     * TODO:
+//                     * ASSUMES THE BACKING LIST (i.e., 'sequence') IS _NOT_ STRUCTURALLY MODIFIED, hence may not work
+//                     * for live traces!
+//                     */
+//                    // TODO: ALSO CHECK TIMING CONSTRAINT
+//                    PcapPacket firstPacket = sequence.get(seqIdx - lowerBound.size());
+//                    PcapPacket lastPacket = sequence.get(seqIdx);
+//                    if (!lastPacket.getTimestamp().isAfter(firstPacket.getTimestamp().plusMillis(mInclusionTimeMillis))) {
+//                        return Optional.of(sequence.subList(seqIdx - lowerBound.size(), seqIdx));
+//                    }
+//                }
+//            } else {
+//                // Mismatch.
+//                if (subseqIdx > 0) {
+//                    /*
+//                     * If we managed to match parts of subsequence, we restart the search for subsequence in sequence at
+//                     * the index of sequence where the current mismatch occurred. I.e., we must reset subseqIdx, but
+//                     * leave seqIdx untouched.
+//                     */
+//                    subseqIdx = 0;
+//                } else {
+//                    /*
+//                     * First packet of subsequence didn't match packet at seqIdx of sequence, so we move forward in
+//                     * sequence, i.e., we continue the search for subsequence in sequence starting at index seqIdx+1 of
+//                     * sequence.
+//                     */
+//                    seqIdx++;
+//                }
+//            }
+//        }
+//        return Optional.empty();
+//    }
+
     /**
      * Given a cluster, produces a pruned version of that cluster. In the pruned version, there are no duplicate cluster
      * members. Two cluster members are considered identical if their packets lengths and packet directions are

diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/TcpReassembler.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/TcpReassembler.java
index e1508758fdcf1590ebdf21ca74e9c43406561526..3b716e19554f91fd011864544721186511b529df 100644 (file)
--- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/TcpReassembler.java
+++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/TcpReassembler.java
@@ -36,6 +36,20 @@ public class TcpReassembler implements PacketListener {
      * Holds <em>terminated</em> {@link Conversation}s.
      */
     private final List<Conversation> mTerminatedConversations = new ArrayList<>();
+               
+               /**
+     * IP of the router's WAN port (if analyzed traffic is captured at the ISP's point of view).
+     */
+    private final String mRouterWanIp;
+               private static final String ROUTER_WAN_IP = "128.195.205.105";
+               
+    public TcpReassembler() {
+        mRouterWanIp = ROUTER_WAN_IP;
+    }
+               
+    public TcpReassembler(String routerWanIp) {
+        mRouterWanIp = routerWanIp;
+    }
 
     @Override
     public void gotPacket(PcapPacket pcapPacket) {
@@ -250,7 +264,7 @@ public class TcpReassembler implements PacketListener {
                 // network, but that obviously won't be a useful strategy for an observer at the WAN port.
                 String srcIp = pcapPacket.get(IpV4Packet.class).getHeader().getSrcAddr().getHostAddress();
                 // TODO: REPLACE THE ROUTER'S IP WITH A PARAMETER!!!
-                boolean clientIsSrc = srcIp.startsWith("10.") || srcIp.startsWith("192.168.") || srcIp.equals("128.195.205.105");
+                boolean clientIsSrc = srcIp.startsWith("10.") || srcIp.startsWith("192.168.") || srcIp.equals(mRouterWanIp);
                 conv = Conversation.fromPcapPacket(pcapPacket, clientIsSrc);
             }
             mOpenConversations.put(conv, conv);