First work on layer 2 sequence matching. Basic functionality seems to work. Cleanup...
authorJanus Varmarken <varmarken@gmail.com>
Sun, 13 Jan 2019 03:58:13 +0000 (19:58 -0800)
committerJanus Varmarken <varmarken@gmail.com>
Sun, 13 Jan 2019 03:58:44 +0000 (19:58 -0800)
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/L2FlowReassembler.java [new file with mode: 0644]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Layer2Flow.java [new file with mode: 0644]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Layer2FlowReassemblerObserver.java [new file with mode: 0644]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/StateMachine.java [new file with mode: 0644]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/AbstractClusterMatcher.java [new file with mode: 0644]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/L2ClusterMatcher.java [new file with mode: 0644]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/Layer2ClusterMatcher.java [new file with mode: 0644]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/Layer2FlowObserver.java [new file with mode: 0644]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/Layer2SequenceMatcher.java [new file with mode: 0644]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java

diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/L2FlowReassembler.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/L2FlowReassembler.java
new file mode 100644 (file)
index 0000000..b09189f
--- /dev/null
@@ -0,0 +1,84 @@
+package edu.uci.iotproject;
+
+import org.pcap4j.core.PacketListener;
+import org.pcap4j.core.PcapPacket;
+import org.pcap4j.packet.EthernetPacket;
+import org.pcap4j.util.MacAddress;
+
+import java.util.*;
+
+/**
+ * Reassembles traffic flows at layer 2, i.e., for each combination of hosts, creates a list of packets exchanged
+ * between said hosts.
+ *
+ * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
+ * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
+ */
+public class L2FlowReassembler implements PacketListener {
+
+    /**
+     * Maps a pair of MAC addresses to the packets exchanged between the two hosts.
+     * The key is the concatenation of the two MAC addresses in hex string format, where the lexicographically smaller
+     * MAC is at the front of the string.
+     */
+    private final Map<String, Layer2Flow> mFlows = new HashMap<>();
+
+    private final List<Layer2FlowReassemblerObserver> mObservers = new ArrayList<>();
+
+    @Override
+    public void gotPacket(PcapPacket packet) {
+        // TODO: update to 802.11 packet...?
+        EthernetPacket ethPkt = packet.get(EthernetPacket.class);
+
+        MacAddress srcAddr = ethPkt.getHeader().getSrcAddr();
+        MacAddress dstAddr = ethPkt.getHeader().getDstAddr();
+
+        String key = keyFromAddresses(srcAddr, dstAddr);
+        // Create a new list if this pair of MAC addresses where not previously encountered and add packet to that list,
+        // or simply add to an existing list if one is present.
+        mFlows.computeIfAbsent(key, k -> {
+            Layer2Flow newFlow = new Layer2Flow(srcAddr, dstAddr);
+            // Inform observers of the new flow
+            mObservers.forEach(o -> o.onNewFlow(this, newFlow));
+            return newFlow;
+        }).addPacket(packet);
+    }
+
+    public void addObserver(Layer2FlowReassemblerObserver observer) {
+        mObservers.add(observer);
+    }
+
+    public void removeObserver(Layer2FlowReassemblerObserver observer) {
+        mObservers.remove(observer);
+    }
+
+    /**
+     * Get the traffic flow between two local endpoints ({@link MacAddress}es).
+     * @param addr1 The first endpoint.
+     * @param addr2 The second endpoint
+     * @return The traffic exchanged between the two endpoints.
+     */
+    public Layer2Flow getFlowForAddresses(MacAddress addr1, MacAddress addr2) {
+        return mFlows.get(keyFromAddresses(addr1, addr2));
+    }
+
+    /**
+     * Get all traffic flows, i.e., a traffic flow for each unique pair of endpoints (MAC addresses).
+     * @return All traffic flows.
+     */
+    public Collection<Layer2Flow> getFlows() {
+        return mFlows.values();
+    }
+
+    /**
+     * Given two {@link MacAddress}es, generates the corresponding key string used in {@link #mFlows}.
+     * @param addr1 The first address.
+     * @param addr2 The second address.
+     * @return the key string used in {@link #mFlows} corresponding to the two addresses.
+     */
+    private String keyFromAddresses(MacAddress addr1, MacAddress addr2) {
+        String addr1Str = addr1.toString();
+        String addr2Str = addr2.toString();
+        return addr1Str.compareTo(addr2Str) < 0 ? addr1Str + addr2Str : addr2Str + addr1Str;
+    }
+}
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Layer2Flow.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Layer2Flow.java
new file mode 100644 (file)
index 0000000..d50a911
--- /dev/null
@@ -0,0 +1,79 @@
+package edu.uci.iotproject;
+
+import edu.uci.iotproject.detection.Layer2FlowObserver;
+import org.pcap4j.core.PcapPacket;
+import org.pcap4j.packet.EthernetPacket;
+import org.pcap4j.util.MacAddress;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * The packets exchanged between two endpoints (MAC addresses).
+ *
+ * @author Janus Varmarken
+ */
+public class Layer2Flow {
+
+    private final MacAddress mEndpoint1;
+    private final MacAddress mEndpoint2;
+
+    private final List<Layer2FlowObserver> mFlowObservers = new ArrayList<>();
+
+    public Layer2Flow(MacAddress endpoint1, MacAddress endpoint2) {
+        mEndpoint1 = endpoint1;
+        mEndpoint2 = endpoint2;
+    }
+
+    public void addFlowObserver(Layer2FlowObserver observer) {
+        mFlowObservers.add(observer);
+    }
+
+    public void removeFlowObserver(Layer2FlowObserver observer) {
+        mFlowObservers.remove(observer);
+    }
+
+    /**
+     * The packets in the flow.
+     */
+    private final List<PcapPacket> mPackets = new ArrayList<>();
+
+    /**
+     * Add a packet to this flow.
+     * @param packet The packet that is to be added to the flow.
+     */
+    public void addPacket(PcapPacket packet) {
+        verifyAddresses(packet);
+        mPackets.add(packet);
+        // Notify flow observers of the new packet
+        mFlowObservers.forEach(o -> o.onNewPacket(this, packet));
+    }
+
+    public List<PcapPacket> getPackets() {
+        return Collections.unmodifiableList(mPackets);
+    }
+
+    private void verifyAddresses(PcapPacket packet) {
+        EthernetPacket ethPkt = packet.get(EthernetPacket.class);
+        MacAddress srcAddr = ethPkt.getHeader().getSrcAddr();
+        MacAddress dstAddr = ethPkt.getHeader().getDstAddr();
+        if ((mEndpoint1.equals(srcAddr) && mEndpoint2.equals(dstAddr)) ||
+                (mEndpoint1.equals(dstAddr) && mEndpoint2.equals(srcAddr))) {
+            // All is good.
+            return;
+        }
+        throw new IllegalArgumentException("Mismatch in MACs: packet does not pertain to this flow");
+    }
+
+}
+
+
+
+/*
+
+
+ Packet stream -> flow reassembler -> flow1, flow2, flow3... -> for each flow, keep a sequence matcher for each sequence of cluster
+
+
+ */
\ No newline at end of file
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Layer2FlowReassemblerObserver.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Layer2FlowReassemblerObserver.java
new file mode 100644 (file)
index 0000000..c9eb2f7
--- /dev/null
@@ -0,0 +1,19 @@
+package edu.uci.iotproject;
+
+/**
+ * For observing a {@link L2FlowReassembler}.
+ *
+ * @author Janus Varmarken
+ */
+public interface Layer2FlowReassemblerObserver {
+
+    /**
+     * Invoked when when a {@link L2FlowReassembler} detects a new flow (i.e., when it encounters traffic between two
+     * MAC addresses that has not previously communicated in the traffic trace).
+     *
+     * @param reassembler The {@link L2FlowReassembler} that detected the new flow.
+     * @param newFlow The new flow.
+     */
+    void onNewFlow(L2FlowReassembler reassembler, Layer2Flow newFlow);
+
+}
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/StateMachine.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/StateMachine.java
new file mode 100644 (file)
index 0000000..dc9e51a
--- /dev/null
@@ -0,0 +1,126 @@
+package edu.uci.iotproject;
+
+import org.jgrapht.Graphs;
+import org.jgrapht.graph.DefaultEdge;
+import org.jgrapht.graph.SimpleDirectedGraph;
+import org.pcap4j.core.PcapPacket;
+
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * TODO add class documentation.
+ *
+ * @author Janus Varmarken
+ */
+public class StateMachine {
+
+
+    private final SimpleDirectedGraph<Vertex, DefaultEdge> mGraph = new SimpleDirectedGraph<>(DefaultEdge.class);
+
+
+    public StateMachine(List<List<PcapPacket>> subCluster) {
+
+        for (List<PcapPacket> seqVariation : subCluster) {
+
+            Vertex currVtx;
+            Vertex prevVtx = null;
+
+            for (int i = 0; i < seqVariation.size(); i++) {
+                // Create new vertex corresponding to this packet of the sequence
+                PcapPacket currPkt = seqVariation.get(i);
+                currVtx = new Vertex(currPkt.getOriginalLength(), i);
+
+
+
+
+
+                mGraph.addVertex(currVtx);
+
+
+
+                if (prevVtx != null) {
+                    // Link vertex representing previous packet of sequence to this vertex.
+                    mGraph.addEdge(prevVtx, currVtx);
+
+                }
+
+                // Current vertex becomes previous vertex for next iteration.
+                prevVtx = currVtx;
+            }
+        }
+
+    }
+
+
+    private Vertex mCurrentState;
+
+//    @Override
+//    public void gotPacket(PcapPacket packet) {
+//        // Generate a vertex corresponding to the received packet.
+//        // We expect a packet at the layer that follows the current state's layer.
+//        Vertex pktVtx = new Vertex(packet.getOriginalLength(), mCurrentState.mLayer + 1);
+//        // Check if such a vertex is present as a successor of the current state
+//        Optional<Vertex> match = Graphs.successorListOf(mGraph, mCurrentState).stream().
+//                filter(v -> v.equals(pktVtx)).findFirst();
+//        // If yes, we move to that new state (new vertex).
+//        match.ifPresent(v -> mCurrentState = v);
+//        // TODO buffer the packets that got us here
+//        // TODO check if we've reached the final layer...
+//
+//    }
+
+
+    /**
+     * Attempts to use {@code packet} to advance this state machine.
+     * @param packet
+     * @return {@code true} if this state machine could progress by consuming {@code packet}, {@code false} otherwise.
+     */
+    public boolean attemptAdvance(PcapPacket packet) {
+        // Generate a vertex corresponding to the received packet.
+        // We expect a packet at the layer that follows the current state's layer.
+        Vertex pktVtx = new Vertex(packet.getOriginalLength(), mCurrentState.mLayer + 1);
+        // Check if such a vertex is present as a successor of the current state
+        Optional<Vertex> match = Graphs.successorListOf(mGraph, mCurrentState).stream().
+                filter(v -> v.equals(pktVtx)).findFirst();
+        if (match.isPresent()) {
+            // If yes, we move to that new state (new vertex).
+            mCurrentState = match.get();
+            // TODO buffer the packet to keep track of what packets got us here (keep track of the match)
+            // TODO check if we've reached the final layer...
+
+            return true;
+        }
+        return false;
+    }
+
+    private static class Vertex {
+
+        // TODO how to include direction of packets here...
+
+        private final int mPktLength;
+        private final int mLayer;
+
+
+        private Vertex(int pktLength, int layer) {
+            mPktLength = pktLength;
+            mLayer = layer;
+        }
+
+
+        @Override
+        public boolean equals(Object obj) {
+            if (!(obj instanceof Vertex)) return false;
+            Vertex that = (Vertex) obj;
+            return that.mPktLength == this.mPktLength && that.mLayer == this.mLayer;
+        }
+
+        @Override
+        public int hashCode() {
+//            return Integer.hashCode(mPktLength);
+            // Hack: use string's hashCode implementation.
+            return (Integer.toString(mPktLength) + " " + Integer.toString(mLayer)).hashCode();
+        }
+
+    }
+}
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/AbstractClusterMatcher.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/AbstractClusterMatcher.java
new file mode 100644 (file)
index 0000000..4c74eb8
--- /dev/null
@@ -0,0 +1,38 @@
+package edu.uci.iotproject.detection;
+
+import edu.uci.iotproject.Conversation;
+import org.pcap4j.core.PcapPacket;
+
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * TODO add class documentation.
+ *
+ * @author Janus Varmarken
+ */
+abstract public class AbstractClusterMatcher {
+
+    protected final List<List<PcapPacket>> mCluster;
+
+
+    protected AbstractClusterMatcher(List<List<PcapPacket>> cluster) {
+        // ===================== PRECONDITION SECTION =====================
+        cluster = Objects.requireNonNull(cluster, "cluster cannot be null");
+        if (cluster.isEmpty() || cluster.stream().anyMatch(inner -> inner.isEmpty())) {
+            throw new IllegalArgumentException("cluster is empty (or contains an empty inner List)");
+        }
+        mCluster = pruneCluster(cluster);
+    }
+
+    /**
+     * Allows subclasses to specify how to prune input cluster provided to the constructor.
+     * @param cluster The input cluster provided to the constructor.
+     * @return The pruned cluster to use in place of the input cluster.
+     */
+    abstract protected List<List<PcapPacket>> pruneCluster(List<List<PcapPacket>> cluster);
+
+    // TODO: move Direction outside Conversation so that this is less confusing.
+//    abstract protected Conversation.Direction[] getPacketDirections(List<PcapPacket> packets);
+
+}
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/L2ClusterMatcher.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/L2ClusterMatcher.java
new file mode 100644 (file)
index 0000000..b673e28
--- /dev/null
@@ -0,0 +1,196 @@
+package edu.uci.iotproject.detection;
+
+import edu.uci.iotproject.Layer2Flow;
+import edu.uci.iotproject.L2FlowReassembler;
+import edu.uci.iotproject.StateMachine;
+import edu.uci.iotproject.util.PcapPacketUtils;
+import org.pcap4j.core.PacketListener;
+import org.pcap4j.core.PcapPacket;
+import org.pcap4j.util.MacAddress;
+
+import java.util.*;
+
+/**
+ * Layer 2 cluster matcher.
+ *
+ * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
+ * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
+ */
+public class L2ClusterMatcher extends AbstractClusterMatcher implements PacketListener {
+
+    private final MacAddress mRouterMac = null;
+    private final MacAddress mPhoneMac = null;
+    private final MacAddress mDeviceMac = null;
+
+    /**
+     * Reassembles traffic flows.
+     */
+    private final L2FlowReassembler mFlowReassembler = new L2FlowReassembler();
+
+    /**
+     * Each inner set holds the possible packet lengths for the packet at the corresponding index in a sequemce, taken
+     * across all sequences in {@link #mCluster}. For example, if the cluster is comprised of the sequences [112, 115]
+     * and [112, 116], the set at index 0 will be {112}, and the set at index 1 will be {115, 116}.
+     */
+    private final List<Set<Integer>> mValidPktLengths;
+
+
+
+    // Maintain one state machine for each layer...?
+    private final StateMachine[] seqMatchers;
+
+    public L2ClusterMatcher(List<List<PcapPacket>> cluster) {
+        super(cluster);
+
+        mValidPktLengths = new ArrayList<>();
+        for (int i = 0; i < mCluster.get(0).size(); i++) {
+            mValidPktLengths.add(new HashSet<>());
+        }
+        for (List<PcapPacket> seqVariation : mCluster) {
+            for (int i = 0; i < seqVariation.size(); i++) {
+                mValidPktLengths.get(i).add(seqVariation.get(i).getOriginalLength());
+            }
+        }
+
+        seqMatchers = new StateMachine[mValidPktLengths.size()];
+    }
+
+    @Override
+    protected List<List<PcapPacket>> pruneCluster(List<List<PcapPacket>> cluster) {
+        return null;
+    }
+
+
+    @Override
+    public void gotPacket(PcapPacket packet) {
+        for (int i = 0; i < seqMatchers.length; i++) {
+            StateMachine sm = seqMatchers[i];
+            if (sm.attemptAdvance(packet)) {
+
+            }
+
+        }
+
+
+
+
+
+
+
+        for (int i = 0; i < mValidPktLengths.size(); i++) {
+            if (mValidPktLengths.get(i).contains(packet.getOriginalLength())) {
+                // This packet length is potentially of interest to state machines that currently expect the i'th packet
+                // of the searched sequence
+
+            }
+        }
+
+
+
+
+        // Forward to flow reassembler
+        mFlowReassembler.gotPacket(packet);
+
+
+
+
+    }
+
+
+    public void performDetection() {
+        for (Layer2Flow flow : mFlowReassembler.getFlows()) {
+            List<PcapPacket> flowPkts = flow.getPackets();
+
+            for (List<PcapPacket> signatureSequence : mCluster) {
+
+            }
+        }
+    }
+
+/*
+    private Optional<List<PcapPacket>> findSubsequenceInSequence(List<PcapPacket> subsequence,
+                                                                 List<PcapPacket> sequence,
+                                                                 boolean[] subsequenceDirections) {
+        if (sequence.size() < subsequence.size()) {
+            // If subsequence is longer, it cannot be contained in sequence.
+            return Optional.empty();
+        }
+        // If packet directions have not been precomputed by calling code, we need to construct them.
+        if (subsequenceDirections == null) {
+            subsequenceDirections = getPacketDirections(subsequence);
+        }
+
+
+
+
+
+
+//        if (sequenceDirections == null) {
+//            sequenceDirections = getPacketDirections(sequence);
+//        }
+
+
+        boolean[] sequenceDirections;
+
+        int subseqIdx = 0;
+        int seqIdx = 0;
+        while (seqIdx < sequence.size()) {
+            if (subseqIdx == 0) {
+                // Every time we (re-)start matching (i.e., when we consider the first element of subsequence), we must
+                // recompute the directions array for the subsequence.size() next elements of sequence so that we can
+                // perform index-wise comparisons of the individual elements of the two direction arrays. If we compute
+                // the directions array for the entire sequence in one go, we may end up with a reversed representation
+                // of the packet directions (i.e. one in which all boolean values in the array are flipped to be the
+                // opposite of what is the expected order) for a subsection of sequence that actually obeys the expected
+                // directions (as defined by the directions array corresponding to subsequence), depending on the packets
+                // that come earlier (as we always use 'true' for the first packet direction of a sequence).
+                int toIndex = Integer.min(seqIdx + subsequence.size(), sequence.size());
+                sequenceDirections = getPacketDirections(sequence.subList(seqIdx, toIndex));
+            }
+
+
+            PcapPacket subseqPkt = subsequence.get(subseqIdx);
+            PcapPacket seqPkt = sequence.get(seqIdx);
+            // We only have a match if packet lengths and directions match.
+            if (subseqPkt.getOriginalLength() == seqPkt.getOriginalLength() &&
+                    subsequenceDirections[subseqIdx] == sequenceDirections[subseqIdx]) {
+                if (subseqIdx > 0) {
+
+                }
+            }
+        }
+    }
+    */
+
+    /**
+     * Returns a boolean array {@code b} such that each entry in {@code b} indicates the direction of the packet at the
+     * corresponding index in {@code pktSequence}. As there is no notion of client and server, we model the
+     * packet directions as simple binary values. The direction of the first packet in {@code pktSequence} (and all
+     * subsequent packets going in the same direction) is denoted using a value of {@code true}, and all packets going
+     * in the opposite direction are denoted using a value of {@code false}.
+     *
+     * @param pktSequence A sequence of packets exchanged between two hosts for which packet directions are to be
+     *                    extracted.
+     * @return The packet directions for {@code pktSequence}.
+     */
+    private boolean[] getPacketDirections(List<PcapPacket> pktSequence) {
+        boolean[] directions = new boolean[pktSequence.size()];
+        for (int i = 0; i < pktSequence.size(); i++) {
+            if (i == 0) {
+                // Special case for first packet: no previous packet to compare against.
+                directions[i] = true;
+            } else {
+                PcapPacket currPkt = pktSequence.get(i);
+                PcapPacket prevPkt = pktSequence.get(i-1);
+                if (PcapPacketUtils.getEthSrcAddr(currPkt).equals(PcapPacketUtils.getEthSrcAddr(prevPkt))) {
+                    // Same direction as previous packet.
+                    directions[i] = directions[i-1];
+                } else {
+                    // Opposite direction of previous packet.
+                    directions[i] = !directions[i-1];
+                }
+            }
+        }
+        return directions;
+    }
+}
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/Layer2ClusterMatcher.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/Layer2ClusterMatcher.java
new file mode 100644 (file)
index 0000000..e0438d6
--- /dev/null
@@ -0,0 +1,148 @@
+package edu.uci.iotproject.detection;
+
+import edu.uci.iotproject.L2FlowReassembler;
+import edu.uci.iotproject.Layer2Flow;
+import edu.uci.iotproject.Layer2FlowReassemblerObserver;
+import edu.uci.iotproject.io.PcapHandleReader;
+import edu.uci.iotproject.util.PrintUtils;
+import org.pcap4j.core.*;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * TODO add class documentation.
+ *
+ * @author Janus Varmarken
+ */
+public class Layer2ClusterMatcher extends AbstractClusterMatcher implements Layer2FlowReassemblerObserver, Layer2FlowObserver {
+
+    public static void main(String[] args) throws PcapNativeException, NotOpenException {
+        final String onSignatureFile = "/Users/varmarken/temp/UCI IoT Project/experiments/training/signatures/tplink-plug/tplink-plug-onSignature-device-side.sig";
+        List<List<List<PcapPacket>>> onSignature = PrintUtils.deserializeSignatureFromFile(onSignatureFile);
+
+
+        L2FlowReassembler flowReassembler = new L2FlowReassembler();
+
+        Layer2ClusterMatcher l2ClusterMatcher = new Layer2ClusterMatcher(onSignature.get(0));
+        flowReassembler.addObserver(l2ClusterMatcher);
+
+        final String inputPcapFile = "/Users/varmarken/temp/UCI IoT Project/experiments/2018-07/tplink/tplink.wlan1.local.pcap";
+
+        PcapHandle handle;
+        try {
+            handle = Pcaps.openOffline(inputPcapFile, PcapHandle.TimestampPrecision.NANO);
+        } catch (PcapNativeException pne) {
+            handle = Pcaps.openOffline(inputPcapFile);
+        }
+        PcapHandleReader reader = new PcapHandleReader(handle, p -> true, flowReassembler);
+        reader.readFromHandle();
+
+
+    }
+
+
+    private final List<Layer2SequenceMatcher> mSeqMatchers;
+
+    public Layer2ClusterMatcher(List<List<PcapPacket>> cluster) {
+        super(cluster);
+        // Setup a sequence matcher for each sequence of the pruned cluster
+        mSeqMatchers = new ArrayList<>();
+        mCluster.forEach(seq -> mSeqMatchers.add(new Layer2SequenceMatcher(seq)));
+
+//        for (int i = 0; i < mCluster.size(); i++) {
+//
+//
+//            mSeqMatchers[i] = new Layer2SequenceMatcher(mCluster.get(i));
+//
+//
+//        }
+    }
+
+//    @Override
+//    public void gotPacket(PcapPacket packet) {
+//        // Forward the packet to all sequence matchers.
+//        for (Layer2SequenceMatcher matcher : mSeqMatchers) {
+//            matcher.gotPacket(packet);
+//        }
+//
+//
+//    }
+
+
+    private final Map<Layer2Flow, List<Layer2SequenceMatcher>> mPerFlowSeqMatchers = new HashMap<>();
+
+    @Override
+    public void onNewPacket(Layer2Flow flow, PcapPacket newPacket) {
+        if (mPerFlowSeqMatchers.get(flow) == null) {
+            // If this is the first time we encounter this flow, we need to set up sequence matchers for it.
+            List<Layer2SequenceMatcher> matchers = new ArrayList<>();
+            mCluster.forEach(seq -> matchers.add(new Layer2SequenceMatcher(seq)));
+            mPerFlowSeqMatchers.put(flow, matchers);
+        }
+        // Buffer for new sequence matchers that will take over the job of observing for the first packet when a
+        // sequence matcher advances beyond the first packet.
+        List<Layer2SequenceMatcher> newSeqMatchers = new ArrayList<>();
+        // Buffer for sequence matchers that have terminated and are to be removed from mPerFlowSeqMatchers.
+        List<Layer2SequenceMatcher> terminatedSeqMatchers = new ArrayList<>();
+        // Present the new packet to all sequence matchers
+        for (Layer2SequenceMatcher sm : mPerFlowSeqMatchers.get(flow)) {
+            boolean matched = sm.matchPacket(newPacket);
+            if (matched && sm.getMatchedPacketsCount() == 1) {
+                // Setup a new sequence matcher that matches from the beginning of the sequence so as to keep
+                // progressing in the sequence matcher that just matched the current packet, while still allowing
+                // for matches of the full sequence in later traffic. This is to accommodate the case where the
+                // first packet of a sequence is detected in an early packet, but where the remaining packets of
+                // that sequence do not appear until way later in time (e.g., if the first packet of the sequence
+                // by chance is generated from traffic unrelated to the trigger traffic).
+                // Note that we must store the new sequence matcher in a buffer and add it outside the loop in order to
+                // prevent concurrent modification exceptions.
+                newSeqMatchers.add(new Layer2SequenceMatcher(sm.getTargetSequence()));
+            }
+            if (matched && sm.getMatchedPacketsCount() == sm.getTargetSequencePacketCount()) {
+                // This sequence matcher has a match of the sequence it was searching for
+                // TODO report it.... for now just do a dummy printout.
+                System.out.println("SEQUENCE MATCHER HAS A MATCH AT " + sm.getMatchedPackets().get(0).getTimestamp());
+                // Mark the sequence matcher for removal. No need to create a replacement one as we do that whenever the
+                // first packet of the sequence is matched (see above).
+                terminatedSeqMatchers.add(sm);
+            }
+        }
+        // Add the new sequence matchers, if any.
+        mPerFlowSeqMatchers.get(flow).addAll(newSeqMatchers);
+        // Remove the terminated sequence matchers, if any.
+        mPerFlowSeqMatchers.get(flow).removeAll(terminatedSeqMatchers);
+    }
+
+
+    @Override
+    protected List<List<PcapPacket>> pruneCluster(List<List<PcapPacket>> cluster) {
+        // Note: we assume that all sequences in the input cluster are of the same length and that their packet
+        // directions are identical.
+        List<List<PcapPacket>> prunedCluster = new ArrayList<>();
+        for (List<PcapPacket> originalClusterSeq : cluster) {
+            boolean alreadyPresent = prunedCluster.stream().anyMatch(pcPkts -> {
+                for (int i = 0; i < pcPkts.size(); i++) {
+                    if (pcPkts.get(i).getOriginalLength() != originalClusterSeq.get(i).getOriginalLength()) {
+                        return false;
+                    }
+                }
+                return true;
+            });
+            if (!alreadyPresent) {
+                // Add the sequence if not already present in the pruned cluster.
+                prunedCluster.add(originalClusterSeq);
+            }
+        }
+        return prunedCluster;
+    }
+
+
+    @Override
+    public void onNewFlow(L2FlowReassembler reassembler, Layer2Flow newFlow) {
+        // Subscribe to the new flow to get updates whenever a new packet pertaining to the flow is processed.
+        newFlow.addFlowObserver(this);
+    }
+}
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/Layer2FlowObserver.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/Layer2FlowObserver.java
new file mode 100644 (file)
index 0000000..05cf9a1
--- /dev/null
@@ -0,0 +1,15 @@
+package edu.uci.iotproject.detection;
+
+import edu.uci.iotproject.Layer2Flow;
+import org.pcap4j.core.PcapPacket;
+
+/**
+ * TODO add class documentation.
+ *
+ * @author Janus Varmarken
+ */
+public interface Layer2FlowObserver {
+
+    void onNewPacket(Layer2Flow flow, PcapPacket newPacket);
+
+}
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/Layer2SequenceMatcher.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/Layer2SequenceMatcher.java
new file mode 100644 (file)
index 0000000..35d628d
--- /dev/null
@@ -0,0 +1,76 @@
+package edu.uci.iotproject.detection;
+
+import edu.uci.iotproject.analysis.TriggerTrafficExtractor;
+import org.pcap4j.core.PcapPacket;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * TODO add class documentation.
+ *
+ * @author Janus Varmarken
+ */
+public class Layer2SequenceMatcher {
+
+    /**
+     * The sequence this {@link Layer2SequenceMatcher} is searching for.
+     */
+    private final List<PcapPacket> mSequence;
+
+    /**
+     * Buffer of actual packets seen so far that match the searched sequence (i.e., constitutes a subsequence of the
+     * searched sequence).
+     */
+    private final List<PcapPacket> mMatchedPackets = new ArrayList<>();
+
+    public Layer2SequenceMatcher(List<PcapPacket> sequence) {
+        mSequence = sequence;
+    }
+
+    public boolean matchPacket(PcapPacket packet) {
+        // The packet we want to match next.
+        PcapPacket expected = mSequence.get(mMatchedPackets.size());
+        // First verify if the received packet has the length we're looking for.
+        if (packet.getOriginalLength() == expected.getOriginalLength()) {
+            // Next apply timing constraints:
+            // - to be a match, the packet must have a later timestamp than any other packet currently matched
+            // - does adding the packet cause the max allowed time between first packet and last packet to be exceeded?
+            if (mMatchedPackets.size() > 0 &&
+                    !packet.getTimestamp().isAfter(mMatchedPackets.get(mMatchedPackets.size()-1).getTimestamp())) {
+                return false;
+            }
+            if (mMatchedPackets.size() > 0 &&
+                    packet.getTimestamp().
+                            isAfter(mMatchedPackets.get(0).getTimestamp().
+                                    plusMillis(TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS))) {
+                // Packet too
+                return false;
+            }
+            // TODO (how to) check directions?
+            // This packet has a length matching next packet of searched sequence, so we store it and advance.
+            mMatchedPackets.add(packet);
+            if (mMatchedPackets.size() == mSequence.size()) {
+                // TODO report (to observers?) that we are done.
+            }
+            return true;
+        }
+        return false;
+    }
+
+    public int getMatchedPacketsCount() {
+        return mMatchedPackets.size();
+    }
+
+    public int getTargetSequencePacketCount() {
+        return mSequence.size();
+    }
+
+    public List<PcapPacket> getTargetSequence() {
+        return mSequence;
+    }
+
+    public List<PcapPacket> getMatchedPackets() {
+        return mMatchedPackets;
+    }
+}
index f03110e..cc958ee 100644 (file)
@@ -6,8 +6,10 @@ import edu.uci.iotproject.analysis.TcpConversationUtils;
 import edu.uci.iotproject.analysis.TriggerTrafficExtractor;
 import org.apache.commons.math3.stat.clustering.Cluster;
 import org.pcap4j.core.PcapPacket;
+import org.pcap4j.packet.EthernetPacket;
 import org.pcap4j.packet.IpV4Packet;
 import org.pcap4j.packet.TcpPacket;
+import org.pcap4j.util.MacAddress;
 
 import java.util.*;
 
@@ -26,6 +28,25 @@ public final class PcapPacketUtils {
      */
     private static final int SIGNATURE_MERGE_THRESHOLD = 5;
 
+
+    /**
+     * Gets the source address of the Ethernet part of {@code packet}.
+     * @param packet The packet for which the Ethernet source address is to be extracted.
+     * @return The source address of the Ethernet part of {@code packet}.
+     */
+    public static MacAddress getEthSrcAddr(PcapPacket packet) {
+        return getEthernetPacketOrThrow(packet).getHeader().getSrcAddr();
+    }
+
+    /**
+     * Gets the destination address of the Ethernet part of {@code packet}.
+     * @param packet The packet for which the Ethernet destination address is to be extracted.
+     * @return The destination address of the Ethernet part of {@code packet}.
+     */
+    public static MacAddress getEthDstAddr(PcapPacket packet) {
+        return getEthernetPacketOrThrow(packet).getHeader().getDstAddr();
+    }
+
     /**
      * Determines if a given {@link PcapPacket} wraps a {@link TcpPacket}.
      * @param packet The {@link PcapPacket} to inspect.
@@ -346,7 +367,7 @@ public final class PcapPacketUtils {
     /**
      * Gets the {@link IpV4Packet} contained in {@code packet}, or throws a {@link NullPointerException} if
      * {@code packet} does not contain an {@link IpV4Packet}.
-     * @param packet A {@link PcapPacket} that is expected to contain a {@link IpV4Packet}.
+     * @param packet A {@link PcapPacket} that is expected to contain an {@link IpV4Packet}.
      * @return The {@link IpV4Packet} contained in {@code packet}.
      * @throws NullPointerException if {@code packet} does not encapsulate an {@link IpV4Packet}.
      */
@@ -354,6 +375,17 @@ public final class PcapPacketUtils {
         return Objects.requireNonNull(packet.get(IpV4Packet.class), "not an IPv4 packet");
     }
 
+    /**
+     * Gets the {@link EthernetPacket} contained in {@code packet}, or throws a {@link NullPointerException} if
+     * {@code packet} does not contain an {@link EthernetPacket}.
+     * @param packet A {@link PcapPacket} that is expected to contain an {@link EthernetPacket}.
+     * @return The {@link EthernetPacket} contained in {@code packet}.
+     * @throws NullPointerException if {@code packet} does not encapsulate an {@link EthernetPacket}.
+     */
+    private static final EthernetPacket getEthernetPacketOrThrow(PcapPacket packet) {
+        return Objects.requireNonNull(packet.get(EthernetPacket.class), "not an Ethernet packet");
+    }
+
     /**
      * Print signatures in {@code List} of {@code List} of {@code List} of {@code PcapPacket} objects.
      *