Merge branch 'master' of https://github.uci.edu/rtrimana/smart_home_traffic
authorrtrimana <rtrimana@uci.edu>
Mon, 14 Jan 2019 18:03:15 +0000 (10:03 -0800)
committerrtrimana <rtrimana@uci.edu>
Mon, 14 Jan 2019 18:03:15 +0000 (10:03 -0800)
29 files changed:
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Conversation.java [deleted file]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FinAckPair.java [deleted file]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FlowPatternFinder.java
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/TcpReassembler.java [deleted file]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TcpConversationUtils.java
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TrafficLabeler.java
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/AbstractPatternComparisonResult.java
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/ComparisonFunctions.java
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/CompleteMatchPatternComparisonResult.java
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/PatternComparisonTask.java
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/seqalignment/ExtractedSequence.java
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/seqalignment/SequenceExtraction.java
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/AbstractClusterMatcher.java [new file with mode: 0644]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/ClusterMatcher.java [deleted file]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/SignatureDetector.java [deleted file]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer2/Layer2ClusterMatcher.java [new file with mode: 0644]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SequenceMatcher.java [new file with mode: 0644]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer3/Layer3ClusterMatcher.java [new file with mode: 0644]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer3/SignatureDetector.java [new file with mode: 0644]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/evaluation/SanitySignatureGenerator.java
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2Flow.java [new file with mode: 0644]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowObserver.java [new file with mode: 0644]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowReassembler.java [new file with mode: 0644]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowReassemblerObserver.java [new file with mode: 0644]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/Conversation.java [new file with mode: 0644]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/FinAckPair.java [new file with mode: 0644]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/TcpReassembler.java [new file with mode: 0644]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java

diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Conversation.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Conversation.java
deleted file mode 100644 (file)
index 490dc4f..0000000
+++ /dev/null
@@ -1,585 +0,0 @@
-package edu.uci.iotproject;
-
-import edu.uci.iotproject.analysis.TcpConversationUtils;
-import edu.uci.iotproject.util.PcapPacketUtils;
-import org.pcap4j.core.PcapPacket;
-import org.pcap4j.packet.IpV4Packet;
-import org.pcap4j.packet.Packet;
-import org.pcap4j.packet.TcpPacket;
-
-import java.util.*;
-
-/**
- * Models a (TCP) conversation/connection/session/flow (packet's belonging to the same session between a client and a
- * server).
- * Holds a list of {@link PcapPacket}s identified as pertaining to the flow. Note that this list is <em>not</em>
- * considered when determining equality of two {@code Conversation} instances in order to allow for a
- * {@code Conversation} to function as a key in data structures such as {@link java.util.Map} and {@link java.util.Set}.
- * See {@link #equals(Object)} for the definition of equality.
- *
- * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
- * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
- */
-public class Conversation {
-
-    /* Begin instance properties */
-    /**
-     * The IP of the host that is considered the client (i.e. the host that initiates the conversation)
-     * in this conversation.
-     */
-    private final String mClientIp;
-
-    /**
-     * The port number used by the host that is considered the client in this conversation.
-     */
-    private final int mClientPort;
-
-    /**
-     * The IP of the host that is considered the server (i.e. is the responder) in this conversation.
-     */
-    private final String mServerIp;
-
-    /**
-     * The port number used by the server in this conversation.
-     */
-    private final int mServerPort;
-
-    /**
-     * The list of packets (with payload) pertaining to this conversation.
-     */
-    private final List<PcapPacket> mPackets;
-
-    /**
-     * If {@link #isTls()} is {@code true}, this list contains the subset of {@link #mPackets} which are TLS Application
-     * Data packets.
-     */
-    private final List<PcapPacket> mTlsApplicationDataPackets;
-
-    /**
-     * Contains the sequence numbers used thus far by the host that is considered the <em>client</em> in this
-     * {@code Conversation}.
-     * Used for filtering out retransmissions.
-     */
-    private final Set<Integer> mSeqNumbersClient;
-
-    /**
-     * Contains the sequence numbers used thus far by the host that is considered the <em>server</em> in this
-     * {@code Conversation}.
-     * Used for filtering out retransmissions.
-     */
-    private final Set<Integer> mSeqNumbersSrv;
-
-    /**
-     * List of SYN packets pertaining to this conversation.
-     */
-    private final List<PcapPacket> mSynPackets;
-
-    /**
-     * List of pairs FINs and their corresponding ACKs associated with this conversation.
-     */
-    private final List<FinAckPair> mFinPackets;
-
-    /**
-     * List of RST packets associated with this conversation.
-     */
-    private final List<PcapPacket> mRstPackets;
-
-    /**
-     * Boolean to mark the packet as Application Data based on the previous packet that reaches MTU
-     */
-    private boolean mApplicationData;
-    /* End instance properties */
-
-    /**
-     * Factory method for creating a {@code Conversation} from a {@link PcapPacket}.
-     * @param pcapPacket The {@code PcapPacket} that wraps a TCP segment for which a {@code Conversation} is to be initiated.
-     * @param clientIsSrc If {@code true}, the source address and source port found in the IP datagram and TCP segment
-     *                    wrapped in the {@code PcapPacket} are regarded as pertaining to the client, and the destination
-     *                    address and destination port are regarded as pertaining to the server---and vice versa if set
-     *                    to {@code false}.
-     * @return A {@code Conversation} initiated with ip:port for client and server according to the direction of the packet.
-     */
-    public static Conversation fromPcapPacket(PcapPacket pcapPacket, boolean clientIsSrc) {
-        IpV4Packet ipPacket = pcapPacket.get(IpV4Packet.class);
-        TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class);
-        String clientIp = clientIsSrc ? ipPacket.getHeader().getSrcAddr().getHostAddress() :
-                ipPacket.getHeader().getDstAddr().getHostAddress();
-        String srvIp = clientIsSrc ? ipPacket.getHeader().getDstAddr().getHostAddress() :
-                ipPacket.getHeader().getSrcAddr().getHostAddress();
-        int clientPort = clientIsSrc ? tcpPacket.getHeader().getSrcPort().valueAsInt() :
-                tcpPacket.getHeader().getDstPort().valueAsInt();
-        int srvPort = clientIsSrc ? tcpPacket.getHeader().getDstPort().valueAsInt() :
-                tcpPacket.getHeader().getSrcPort().valueAsInt();
-        return new Conversation(clientIp, clientPort, srvIp, srvPort);
-    }
-
-    /**
-     * Constructs a new {@code Conversation}.
-     * @param clientIp The IP of the host that is considered the client (i.e. the host that initiates the conversation)
-     *                 in the conversation.
-     * @param clientPort The port number used by the client for the conversation.
-     * @param serverIp The IP of the host that is considered the server (i.e. is the responder) in the conversation.
-     * @param serverPort The port number used by the server for the conversation.
-     */
-    public Conversation(String clientIp, int clientPort, String serverIp, int serverPort) {
-        this.mClientIp = clientIp;
-        this.mClientPort = clientPort;
-        this.mServerIp = serverIp;
-        this.mServerPort = serverPort;
-        this.mPackets = new ArrayList<>();
-        this.mTlsApplicationDataPackets = new ArrayList<>();
-        this.mSeqNumbersClient = new HashSet<>();
-        this.mSeqNumbersSrv = new HashSet<>();
-        this.mSynPackets = new ArrayList<>();
-        this.mFinPackets = new ArrayList<>();
-        this.mRstPackets = new ArrayList<>();
-        this.mApplicationData = false;
-    }
-
-    /**
-     * Add a packet to the list of packets associated with this conversation.
-     * @param packet The packet that is to be added to (associated with) this conversation.
-     * @param ignoreRetransmissions Boolean value indicating if retransmissions should be ignored.
-     *                              If set to {@code true}, {@code packet} will <em>not</em> be added to the
-     *                              internal list of packets pertaining to this {@code Conversation}
-     *                              <em>iff</em> the sequence number of {@code packet} was already
-     *                              seen in a previous packet.
-     */
-    public void addPacket(PcapPacket packet, boolean ignoreRetransmissions) {
-        // Precondition: verify that packet does indeed pertain to conversation.
-        onAddPrecondition(packet);
-        if (ignoreRetransmissions && isRetransmission(packet)) {
-            // Packet is a retransmission. Ignore it.
-            return;
-        }
-        // Select direction-dependent set of sequence numbers seen so far and update it with sequence number of new packet.
-        addSeqNumber(packet);
-        // Finally add packet to list of packets pertaining to this conversation.
-        mPackets.add(packet);
-        // Preserve order of packets in list: sort according to timestamp.
-        if (mPackets.size() > 1 &&
-                mPackets.get(mPackets.size()-1).getTimestamp().isBefore(mPackets.get(mPackets.size()-2).getTimestamp())) {
-            Collections.sort(mPackets, (o1, o2) -> {
-                if (o1.getTimestamp().isBefore(o2.getTimestamp())) { return -1; }
-                else if (o2.getTimestamp().isBefore(o1.getTimestamp())) { return 1; }
-                else { return 0; }
-            });
-        }
-        // If TLS, inspect packet to see if it's a TLS Application Data packet, and if so add it to the list of TLS
-        // Application Data packets.
-        if (isTls()) {
-            TcpPacket tcpPacket = packet.get(TcpPacket.class);
-            Packet tcpPayload = tcpPacket.getPayload();
-            if (tcpPayload == null) {
-                return;
-            }
-            byte[] rawPayload = tcpPayload.getRawData();
-            // The SSL record header is at the front of the payload and is 5 bytes long.
-            // The SSL record header type field (the first byte) is set to 23 if it is an Application Data packet.
-            if (rawPayload != null && rawPayload.length >= 5) {
-                if (rawPayload[0] == 23) {
-                    mTlsApplicationDataPackets.add(packet);
-                    // Consider the following packet a data packet if this packet's size == MTU size 1448
-                    if (rawPayload.length >= 1448)
-                        mApplicationData = true;
-                } else if (rawPayload[0] == 20) {
-                    // Do nothing for now - CHANGE_CIPHER_SPEC
-                } else if (rawPayload[0] == 21) {
-                    // Do nothing for now - ALERT
-                } else if (rawPayload[0] == 22) {
-                    // Do nothing for now - HANDSHAKE
-                } else {
-                    // If it is TLS with payload, but rawPayload[0] != 23
-                    if (mApplicationData == true) {
-                        // It is a continuation of the previous packet if the previous packet reaches MTU size 1448 and
-                        // it is not either type 20, 21, or 22
-                        mTlsApplicationDataPackets.add(packet);
-                        if (rawPayload.length < 1448)
-                            mApplicationData = false;
-                    }
-                }
-            }
-        }
-    }
-
-    /**
-     * Get a list of packets pertaining to this {@code Conversation}.
-     * The returned list is a read-only list.
-     * @return the list of packets pertaining to this {@code Conversation}.
-     */
-    public List<PcapPacket> getPackets() {
-        // Return read-only view to prevent external code from manipulating internal state (preserve invariant).
-        return Collections.unmodifiableList(mPackets);
-    }
-
-    /**
-     * Records a TCP SYN packet as pertaining to this conversation (adds it to the the internal list).
-     * Attempts to add duplicate SYN packets will be ignored, and the caller is made aware of the attempt to add a
-     * duplicate by the return value being {@code false}.
-     *
-     * @param synPacket A {@link PcapPacket} wrapping a TCP SYN packet.
-     * @return {@code true} if the packet was successfully added to this {@code Conversation}, {@code false} otherwise.
-     */
-    public boolean addSynPacket(PcapPacket synPacket) {
-        onAddPrecondition(synPacket);
-        final IpV4Packet synPacketIpSection = synPacket.get(IpV4Packet.class);
-        final TcpPacket synPacketTcpSection = synPacket.get(TcpPacket.class);
-        if (synPacketTcpSection == null || !synPacketTcpSection.getHeader().getSyn()) {
-            throw new IllegalArgumentException("Not a SYN packet.");
-        }
-        // We are only interested in recording one copy of the two SYN packets (one SYN packet in each direction), i.e.,
-        // we want to discard retransmitted SYN packets.
-        if (mSynPackets.size() >= 2) {
-            return false;
-        }
-        // Check the set of recorded SYN packets to see if we have already recorded a SYN packet going in the same
-        // direction as the packet given in the argument.
-        boolean matchingPrevSyn = mSynPackets.stream().anyMatch(p -> {
-            IpV4Packet pIp = p.get(IpV4Packet.class);
-            TcpPacket pTcp = p.get(TcpPacket.class);
-            boolean srcAddrMatch = synPacketIpSection.getHeader().getSrcAddr().getHostAddress().
-                    equals(pIp.getHeader().getSrcAddr().getHostAddress());
-            boolean dstAddrMatch = synPacketIpSection.getHeader().getDstAddr().getHostAddress().
-                    equals(pIp.getHeader().getDstAddr().getHostAddress());
-            boolean srcPortMatch = synPacketTcpSection.getHeader().getSrcPort().valueAsInt() ==
-                    pTcp.getHeader().getSrcPort().valueAsInt();
-            boolean dstPortMatch = synPacketTcpSection.getHeader().getDstPort().valueAsInt() ==
-                    pTcp.getHeader().getDstPort().valueAsInt();
-            return srcAddrMatch && dstAddrMatch && srcPortMatch && dstPortMatch;
-        });
-        if (matchingPrevSyn) {
-            return false;
-        }
-        // Update direction-dependent set of sequence numbers and record/log packet.
-        addSeqNumber(synPacket);
-        return mSynPackets.add(synPacket);
-
-        /*
-        mSynPackets.stream().anyMatch(p -> {
-            IpV4Packet pIp = p.get(IpV4Packet.class);
-            TcpPacket pTcp = p.get(TcpPacket.class);
-            boolean srcAddrMatch = synPacketIpSection.getHeader().getSrcAddr().getHostAddress().
-                    equals(pIp.getHeader().getSrcAddr().getHostAddress());
-            boolean dstAddrMatch = synPacketIpSection.getHeader().getDstAddr().getHostAddress().
-                    equals(pIp.getHeader().getDstAddr().getHostAddress());
-            boolean srcPortMatch = synPacketTcpSection.getHeader().getSrcPort().valueAsInt() ==
-                    pTcp.getHeader().getSrcPort().valueAsInt();
-            boolean dstPortMatch = synPacketTcpSection.getHeader().getDstPort().value() ==
-                    pTcp.getHeader().getDstPort().value();
-
-            boolean fourTupleMatch = srcAddrMatch && dstAddrMatch && srcPortMatch && dstPortMatch;
-
-            boolean seqNoMatch = synPacketTcpSection.getHeader().getSequenceNumber() ==
-                    pTcp.getHeader().getSequenceNumber();
-
-            if (fourTupleMatch && !seqNoMatch) {
-                // If the four tuple that identifies the conversation matches, but the sequence number is different,
-                // it means that this SYN packet is, in fact, an attempt to establish a **new** connection, and hence
-                // the given packet is NOT part of this conversation, even though the ip:port combinations are (by
-                // chance) selected such that they match this conversation.
-                throw new IllegalArgumentException("Attempt to add SYN packet that belongs to a different conversation " +
-                        "(which is identified by the same four tuple as this conversation)");
-            }
-            return fourTupleMatch && seqNoMatch;
-        });
-        */
-    }
-
-    /**
-     * Get a list of SYN packets pertaining to this {@code Conversation}.
-     * The returned list is a read-only list.
-     * @return the list of SYN packets pertaining to this {@code Conversation}.
-     */
-    public List<PcapPacket> getSynPackets() {
-        return Collections.unmodifiableList(mSynPackets);
-    }
-
-    /**
-     * Adds a TCP FIN packet to the list of TCP FIN packets associated with this conversation.
-     * @param finPacket The TCP FIN packet that is to be added to (associated with) this conversation.
-     */
-    public void addFinPacket(PcapPacket finPacket) {
-        // Precondition: verify that packet does indeed pertain to conversation.
-        onAddPrecondition(finPacket);
-        // TODO: should call addSeqNumber here?
-        addSeqNumber(finPacket);
-        mFinPackets.add(new FinAckPair(finPacket));
-    }
-
-    /**
-     * Attempt to ACK any FIN packets held by this conversation.
-     * @param ackPacket The ACK for a FIN previously added to this conversation.
-     */
-    public void attemptAcknowledgementOfFin(PcapPacket ackPacket) {
-        // Precondition: verify that the packet pertains to this conversation.
-        onAddPrecondition(ackPacket);
-        // Mark unack'ed FIN(s) that this ACK matches as ACK'ed (there might be more than one in case of retransmissions..?)
-        mFinPackets.replaceAll(finAckPair -> !finAckPair.isAcknowledged() && finAckPair.isCorrespondingAckPacket(ackPacket) ? new FinAckPair(finAckPair.getFinPacket(), ackPacket) : finAckPair);
-    }
-
-    /**
-     * Retrieves an unmodifiable view of the list of {@link FinAckPair}s associated with this {@code Conversation}.
-     * @return an unmodifiable view of the list of {@link FinAckPair}s associated with this {@code Conversation}.
-     */
-    public List<FinAckPair> getFinAckPairs() {
-        return Collections.unmodifiableList(mFinPackets);
-    }
-
-    /**
-     * Get if this {@code Conversation} is considered to have been gracefully shut down.
-     * A {@code Conversation} has been gracefully shut down if it contains a FIN+ACK pair for both directions
-     * (client to server, and server to client).
-     * @return {@code true} if the connection has been gracefully shut down, false otherwise.
-     */
-    public boolean isGracefullyShutdown() {
-        //  The conversation has been gracefully shut down if we have recorded a FIN from both the client and the server which have both been ack'ed.
-        return mFinPackets.stream().anyMatch(finAckPair -> finAckPair.isAcknowledged() && PcapPacketUtils.isSource(finAckPair.getFinPacket(), mClientIp, mClientPort)) &&
-                mFinPackets.stream().anyMatch(finAckPair -> finAckPair.isAcknowledged() && PcapPacketUtils.isSource(finAckPair.getFinPacket(), mServerIp, mServerPort));
-    }
-
-    /**
-     * Add a TCP segment for which the RST flag is set to this {@code Conversation}.
-     * @param packet A {@link PcapPacket} wrapping a TCP segment pertaining to this {@code Conversation} for which the
-     *               RST flag is set.
-     */
-    public void addRstPacket(PcapPacket packet) {
-        /*
-         * TODO:
-         * When now also keeping track of RST packets, should we also...?
-         * 1) Prevent later packets from being added once a RST segment has been added?
-         * 2) Extend 'isGracefullyShutdown()' to also consider RST segments, or add another method, 'isShutdown()' that
-         *    both considers FIN/ACK (graceful) as well as RST (abrupt/"ungraceful") shutdown?
-         * 3) Should it be impossible to associate more than one RST segment with each Conversation?
-         */
-        onAddPrecondition(packet);
-        TcpPacket tcpPacket = packet.get(TcpPacket.class);
-        if (tcpPacket == null || !tcpPacket.getHeader().getRst()) {
-            throw new IllegalArgumentException("not a RST packet");
-        }
-        mRstPackets.add(packet);
-    }
-
-    /**
-     * Get the TCP segments pertaining to this {@code Conversation} for which it was detected that the RST flag is set.
-     * @return the TCP segments pertaining to this {@code Conversation} for which it was detected that the RST flag is
-     *         set.
-     */
-    public List<PcapPacket> getRstPackets() {
-        return Collections.unmodifiableList(mRstPackets);
-    }
-
-    // =========================================================================================================
-    // We simply reuse equals and hashCode methods of String.class to be able to use this class as a key
-    // in a Map.
-
-    /**
-     * <em>Note:</em> currently, equality is determined based on pairwise equality of the elements of the four tuple
-     * ({@link #mClientIp}, {@link #mClientPort}, {@link #mServerIp}, {@link #mServerPort}) for {@code this} and
-     * {@code obj}.
-     * @param obj The object to test for equality with {@code this}.
-     * @return {@code true} if {@code obj} is considered equal to {@code this} based on the definition of equality given above.
-     */
-    @Override
-    public boolean equals(Object obj) {
-        return obj instanceof Conversation && this.toString().equals(obj.toString());
-    }
-
-    @Override
-    public int hashCode() {
-        return toString().hashCode();
-    }
-    // =========================================================================================================
-
-    @Override
-    public String toString() {
-        return String.format("%s:%d %s:%d", mClientIp, mClientPort, mServerIp, mServerPort);
-    }
-
-    /**
-     * Invoke to verify that the precondition holds when a caller attempts to add a packet to this {@code Conversation}.
-     * An {@link IllegalArgumentException} is thrown if the precondition is violated.
-     * @param packet the packet to be added to this {@code Conversation}
-     */
-    private void onAddPrecondition(PcapPacket packet) {
-        // Apply precondition to preserve class invariant: all packets in mPackets must match the 4 tuple that
-        // defines the conversation.
-        IpV4Packet ipPacket = Objects.requireNonNull(packet.get(IpV4Packet.class));
-        // For now we only support TCP flows.
-        TcpPacket tcpPacket = Objects.requireNonNull(packet.get(TcpPacket.class));
-        String ipSrc = ipPacket.getHeader().getSrcAddr().getHostAddress();
-        String ipDst = ipPacket.getHeader().getDstAddr().getHostAddress();
-        int srcPort = tcpPacket.getHeader().getSrcPort().valueAsInt();
-        int dstPort = tcpPacket.getHeader().getDstPort().valueAsInt();
-        String clientIp, serverIp;
-        int clientPort, serverPort;
-        if (ipSrc.equals(mClientIp)) {
-            clientIp = ipSrc;
-            clientPort = srcPort;
-            serverIp = ipDst;
-            serverPort = dstPort;
-        } else {
-            clientIp = ipDst;
-            clientPort = dstPort;
-            serverIp = ipSrc;
-            serverPort = srcPort;
-        }
-        if (!(clientIp.equals(mClientIp) && clientPort == mClientPort &&
-                serverIp.equals(mServerIp) && serverPort == mServerPort)) {
-            throw new IllegalArgumentException(
-                    String.format("Attempt to add packet that does not pertain to %s",
-                            Conversation.class.getSimpleName()));
-        }
-    }
-
-    /**
-     * <p>
-     *      Determines if the TCP packet contained in {@code packet} is a retransmission of a previously seen (logged)
-     *      packet.
-     * </p>
-     *
-     * <b>
-     *     TODO:
-     *     the current implementation, which uses a set of previously seen sequence numbers, will consider a segment
-     *     with a reused sequence number---occurring as a result of sequence number wrap around for a very long-lived
-     *     connection---as a retransmission (and may therefore end up discarding it even though it is in fact NOT a
-     *     retransmission). Ideas?
-     * </b>
-     *
-     * @param packet The packet.
-     * @return {@code true} if {@code packet} was determined to be a retransmission, {@code false} otherwise.
-     */
-    public boolean isRetransmission(PcapPacket packet) {
-        // Extract sequence number.
-        int seqNo = packet.get(TcpPacket.class).getHeader().getSequenceNumber();
-        switch (getDirection(packet)) {
-            case CLIENT_TO_SERVER:
-                return mSeqNumbersClient.contains(seqNo);
-            case SERVER_TO_CLIENT:
-                return mSeqNumbersSrv.contains(seqNo);
-            default:
-                throw new AssertionError(String.format("Unexpected value of enum '%s'",
-                        Direction.class.getSimpleName()));
-        }
-    }
-
-    /**
-     * <p>
-     *     Is this {@code Conversation} a TLS session?
-     * </p>
-     *
-     * <em>Note: the current implementation simply examines the port number(s) for 443; it does <b>not</b> verify if the
-     * application data is indeed encrypted.</em>
-     *
-     * @return {@code true} if this {@code Conversation} is interpreted as a TLS session, {@code false} otherwise.
-     */
-    public boolean isTls() {
-        /*
-         * TODO:
-         * - may want to change this to be "return mServerPort == 443 || mClientPort == 443;" in order to also detect
-         *   TLS in those cases where it is not possible to correctly label who is the client and who is the server,
-         *   i.e., when the trace does not contain the SYN/SYNACK exchange.
-         * - current implementation relies on the server using the conventional TLS port number; may instead want to
-         *   inspect the first 4 bytes of each potential TLS packet to see if they match the SSL record header.
-         *
-         * 08/31/18: Added unconvetional TLS ports used by WeMo plugs and LiFX bulb.
-         * 09/20/18: Moved hardcoded ports to other class to allow other classes to query the set of TLS ports.
-         */
-        return TcpConversationUtils.isTlsPort(mServerPort);
-    }
-
-    /**
-     * If this {@code Conversation} is backing a TLS session (i.e., if the value of {@link #isTls()} is {@code true}),
-     * get the packets labeled as TLS Application Data packets. This is a subset of the full set of payload-carrying
-     * packets (as returned by {@link #getPackets()}). An exception is thrown if this method is invoked on a
-     * {@code Conversation} for which {@link #isTls()} returns {@code false}.
-     *
-     * @return A list containing exactly those packets that could be identified as TLS Application Data packets (through
-     *         inspecting of the SSL record header). The list may be empty, if no TLS application data packets have been
-     *         recorded for this {@code Conversation}.
-     */
-    public List<PcapPacket> getTlsApplicationDataPackets() {
-        if (!isTls()) {
-            throw new NoSuchElementException("cannot get TLS Application Data packets for non-TLS TCP conversation");
-        }
-        return Collections.unmodifiableList(mTlsApplicationDataPackets);
-    }
-
-    /**
-     * Extracts the TCP sequence number from {@code packet} and adds it to the proper set of sequence numbers by
-     * analyzing the direction of the packet.
-     * @param packet A TCP packet (wrapped in a {@code PcapPacket}) that was added to this conversation and whose
-     *               sequence number is to be recorded as seen.
-     */
-    private void addSeqNumber(PcapPacket packet) {
-        // Note: below check is redundant if client code is correct as the call to check the precondition should already
-        // have been made by the addXPacket method that invokes this method. As such, the call below may be removed in
-        // favor of speed, but the improvement will be minor, hence the added safety may be worth it.
-        onAddPrecondition(packet);
-        // Extract sequence number.
-        int seqNo = packet.get(TcpPacket.class).getHeader().getSequenceNumber();
-        // Determine direction of packet and add packet's sequence number to corresponding set of sequence numbers.
-        switch (getDirection(packet)) {
-            case CLIENT_TO_SERVER:
-                // Client to server packet.
-                mSeqNumbersClient.add(seqNo);
-                break;
-            case SERVER_TO_CLIENT:
-                // Server to client packet.
-                mSeqNumbersSrv.add(seqNo);
-                break;
-            default:
-                throw new AssertionError(String.format("Unexpected value of enum '%s'",
-                        Direction.class.getSimpleName()));
-        }
-    }
-
-    /**
-     * Determine the direction of {@code packet}. An {@link IllegalArgumentException} is thrown if {@code packet} does
-     * not pertain to this conversation.
-     *
-     * @param packet The packet whose direction is to be determined.
-     * @return A {@link Direction} indicating the direction of the packet.
-     */
-    public Direction getDirection(PcapPacket packet) {
-        IpV4Packet ipPacket = packet.get(IpV4Packet.class);
-        String ipSrc = ipPacket.getHeader().getSrcAddr().getHostAddress();
-        String ipDst = ipPacket.getHeader().getDstAddr().getHostAddress();
-        // Determine direction of packet.
-        if (ipSrc.equals(mClientIp) && ipDst.equals(mServerIp)) {
-            // Client to server packet.
-            return Direction.CLIENT_TO_SERVER;
-        } else if (ipSrc.equals(mServerIp) && ipDst.equals(mClientIp)) {
-            // Server to client packet.
-            return Direction.SERVER_TO_CLIENT;
-        } else {
-            throw new IllegalArgumentException("getDirection: packet not related to " + getClass().getSimpleName());
-        }
-    }
-
-    /**
-     * Utility enum for expressing the direction of a packet pertaining to this {@code Conversation}.
-     */
-    public enum Direction {
-
-        CLIENT_TO_SERVER {
-            @Override
-            public String toCompactString() {
-                return "*";
-            }
-        },
-        SERVER_TO_CLIENT {
-            @Override
-            public String toCompactString() {
-                return "";
-            }
-        };
-
-        /**
-         * Get a compact string representation of this {@code Direction}.
-         * @return a compact string representation of this {@code Direction}.
-         */
-        abstract public String toCompactString();
-
-    }
-
-}
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FinAckPair.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FinAckPair.java
deleted file mode 100644 (file)
index d4451f3..0000000
+++ /dev/null
@@ -1,134 +0,0 @@
-package edu.uci.iotproject;
-
-import org.pcap4j.core.PcapPacket;
-import org.pcap4j.packet.IpV4Packet;
-import org.pcap4j.packet.TcpPacket;
-
-/**
- * Groups a FIN packet and its corresponding ACK packet. <b>Immutable and thread safe</b>.
- *
- * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
- * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
- */
-public class FinAckPair {
-
-    private final PcapPacket mFinPacket;
-    private final PcapPacket mCorrespondingAckPacket;
-
-    /**
-     * Constructs a {@code FinAckPair} given a FIN packet.
-     * The corresponding ACK packet field is set to {@code null}.
-     * @param finPacket A FIN packet.
-     */
-    public FinAckPair(PcapPacket finPacket) {
-        if (!finPacket.get(TcpPacket.class).getHeader().getFin()) {
-            throw new IllegalArgumentException("not a FIN packet");
-        }
-        mFinPacket = finPacket;
-        mCorrespondingAckPacket = null;
-    }
-
-    /**
-     * Constructs a {@code FinAckPair} given a FIN and an ACK packet.
-     * @param finPacket A FIN packet.
-     * @param correspondingAckPacket The ACK packet corresponding to {@code finPacket}.
-     */
-    public FinAckPair(PcapPacket finPacket, PcapPacket correspondingAckPacket) {
-        // Enforce class invariant, i.e. that the FIN and ACK are related.
-        // Note that it is indirectly checked whether finPacket is indeed a FIN packet
-        // as isCorrespondingAckPacket calls the single parameter constructor.
-        if (!FinAckPair.isCorrespondingAckPacket(finPacket, correspondingAckPacket)) {
-            throw new IllegalArgumentException("FIN and ACK not related");
-        }
-        mFinPacket = finPacket;
-        mCorrespondingAckPacket = correspondingAckPacket;
-    }
-
-    /**
-     * Get the FIN packet of this pair.
-     * @return the FIN packet of this pair.
-     */
-    public PcapPacket getFinPacket() {
-        return mFinPacket;
-    }
-
-    /**
-     * Get the corresponding ACK packet of this pair, if any.
-     * @return the corresponding ACK packet of this pair, if any.
-     */
-    public PcapPacket getCorrespondingAckPacket() {
-        return mCorrespondingAckPacket;
-    }
-
-    /**
-     * Was the FIN in this {@code FinAckPair} acknowledged?
-     *
-     * @return {@code true} if the corresponding ACK has been set in this {@code FinAckPair}.
-     */
-    public boolean isAcknowledged() {
-        return mFinPacket != null && mCorrespondingAckPacket != null;
-    }
-
-    /**
-     * Checks if a given packet is an ACK corresponding to the FIN packet in this {@code FinAckPair}.
-     * @return {@code true} if {@code packet} is an ACK that corresponds to the FIN in this pair, {@code false} otherwise.
-     */
-    public boolean isCorrespondingAckPacket(PcapPacket packet) {
-        IpV4Packet inputIpPacket = packet.get(IpV4Packet.class);
-        TcpPacket inputTcpPacket = packet.get(TcpPacket.class);
-        if (inputIpPacket == null || inputTcpPacket == null || !inputTcpPacket.getHeader().getAck()) {
-            return false;
-        }
-
-        IpV4Packet finIpPacket = mFinPacket.get(IpV4Packet.class);
-        TcpPacket finTcpPacket = mFinPacket.get(TcpPacket.class);
-
-        // Extract (srcIp:port,dstIp:port) for input and member (FIN) packets.
-        String inputPacketIpSrc = inputIpPacket.getHeader().getSrcAddr().getHostAddress();
-        String inputPacketIpDst = inputIpPacket.getHeader().getDstAddr().getHostAddress();
-        int inputPacketPortSrc = inputTcpPacket.getHeader().getSrcPort().valueAsInt();
-        int inputPacketPortDst = inputTcpPacket.getHeader().getDstPort().valueAsInt();
-        String finPacketIpSrc = finIpPacket.getHeader().getSrcAddr().getHostAddress();
-        String finPacketIpDst = finIpPacket.getHeader().getDstAddr().getHostAddress();
-        int finPacketPortSrc = finTcpPacket.getHeader().getSrcPort().valueAsInt();
-        int finPacketPortDst = finTcpPacket.getHeader().getDstPort().valueAsInt();
-
-        // For the two packets to be related, the dst of one must be the src of the other.
-        // Split into multiple if statements for readability. First check IP fields, then ports.
-        if (!(inputPacketIpDst.equals(finPacketIpSrc) && finPacketIpDst.equals(inputPacketIpSrc))) {
-            return false;
-        }
-        if (!(inputPacketPortDst == finPacketPortSrc && finPacketPortDst == inputPacketPortSrc)) {
-            return false;
-        }
-
-        // Packets are (most likely) related (part of same conversation/stream).
-        // Now all that is left for us to check is if the sequence numbers match.
-        // Note: recall that the FIN packet advances the seq numbers by 1,
-        // so the ACK number will be one larger than the seq. number in the FIN packet.
-        return inputTcpPacket.getHeader().getAcknowledgmentNumber() == finTcpPacket.getHeader().getSequenceNumber() + 1;
-    }
-
-    /**
-     * Static method to check if two given packets are a FIN and the corresponding ACK packet.
-     * The purpose of this method is a workaround to enforce the class invariant in the two parameter constructor.
-     * Specifically, the following should be avoided:
-     * <pre>
-     *     public FinAckPair(PcapPacket finPacket, PcapPacket correspondingAckPacket) {
-     *         mFinPacket = finPacket;
-     *         // Below line is considered bad practice as the object has not been fully initialized at this stage.
-     *         if (!this.isCorrespondingAckPacket(correspondingAckPacket)) {
-     *             // ... throw exception
-     *         }
-     *     }
-     * </pre>
-     * @param finPacket The FIN packet.
-     * @param ackPacket The ACK packet that is to be checked if it corresponds to the given FIN packet.
-     * @return {@code true} if the ACK corresponds to the FIN, {@code false} otherwise.
-     */
-    private static boolean isCorrespondingAckPacket(PcapPacket finPacket, PcapPacket ackPacket) {
-        FinAckPair tmp = new FinAckPair(finPacket);
-        return tmp.isCorrespondingAckPacket(ackPacket);
-    }
-
-}
index 1d0a9ba85a03f8bf7f79663a5f3928a5f15dec59..c384852b4c1d91d5bb4bd6024c3d9e17ad26d6d7 100644 (file)
@@ -3,6 +3,7 @@ package edu.uci.iotproject;
 import edu.uci.iotproject.comparison.ComparisonFunctions;
 import edu.uci.iotproject.comparison.CompleteMatchPatternComparisonResult;
 import edu.uci.iotproject.comparison.PatternComparisonTask;
+import edu.uci.iotproject.trafficreassembly.layer3.Conversation;
 import org.pcap4j.core.NotOpenException;
 import org.pcap4j.core.PcapHandle;
 import org.pcap4j.core.PcapNativeException;
@@ -15,7 +16,6 @@ import java.io.*;
 import java.net.UnknownHostException;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
-import java.time.temporal.ChronoField;
 import java.util.*;
 import java.util.concurrent.*;
 
index 5b184d2857780a28ad04cf437b59880d25689425..81eb3b0b9a96c833ae2db2870a4cca21316938da 100644 (file)
@@ -3,10 +3,9 @@ package edu.uci.iotproject;
 import static edu.uci.iotproject.analysis.UserAction.Type;
 
 import edu.uci.iotproject.analysis.*;
-import edu.uci.iotproject.comparison.seqalignment.ExtractedSequence;
-import edu.uci.iotproject.comparison.seqalignment.SequenceAlignment;
-import edu.uci.iotproject.comparison.seqalignment.SequenceExtraction;
 import edu.uci.iotproject.io.TriggerTimesFileReader;
+import edu.uci.iotproject.trafficreassembly.layer3.Conversation;
+import edu.uci.iotproject.trafficreassembly.layer3.TcpReassembler;
 import edu.uci.iotproject.util.PcapPacketUtils;
 import edu.uci.iotproject.util.PrintUtils;
 import org.apache.commons.math3.stat.clustering.Cluster;
@@ -15,8 +14,6 @@ import org.pcap4j.core.*;
 import org.pcap4j.packet.namednumber.DataLinkType;
 
 import java.io.EOFException;
-import java.io.File;
-import java.io.PrintWriter;
 import java.net.UnknownHostException;
 import java.time.Instant;
 import java.util.*;
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/TcpReassembler.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/TcpReassembler.java
deleted file mode 100644 (file)
index 72f0c1c..0000000
+++ /dev/null
@@ -1,260 +0,0 @@
-package edu.uci.iotproject;
-
-import org.pcap4j.core.PacketListener;
-import org.pcap4j.core.PcapPacket;
-import org.pcap4j.packet.*;
-
-import java.util.*;
-
-/**
- * Reassembles TCP conversations (streams).
- * <b>Note: current version only supports TCP over IPv4.</b>
- *
- * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
- * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
- */
-public class TcpReassembler implements PacketListener {
-
-    /**
-     * Holds <em>open</em> {@link Conversation}s, i.e., {@code Conversation}s that have <em>not</em> been detected as
-     * (gracefully) terminated based on the set of packets observed thus far.
-     * A {@link Conversation} is moved to {@link #mTerminatedConversations} if it can be determined that it is has
-     * terminated. Termination can be detected by a) observing two {@link FinAckPair}s, one in each direction, (graceful
-     * termination, see {@link Conversation#isGracefullyShutdown()}) or b) by observing a SYN packet that matches the
-     * four tuple of an existing {@code Conversation}, but which holds a <em>different</em> sequence number than the
-     * same-direction SYN packet recorded for the {@code Conversation}.
-     * <p>
-     * Note that due to limitations of the {@link Set} interface (specifically, there is no {@code get(T t)} method),
-     * we have to resort to a {@link Map} (in which keys map to themselves) to "mimic" a set with {@code get(T t)}
-     * functionality.
-     *
-     * @see <a href="https://stackoverflow.com/questions/7283338/getting-an-element-from-a-set">this question on StackOverflow.com</a>
-     */
-    private final Map<Conversation, Conversation> mOpenConversations = new HashMap<>();
-
-    /**
-     * Holds <em>terminated</em> {@link Conversation}s.
-     */
-    private final List<Conversation> mTerminatedConversations = new ArrayList<>();
-
-    @Override
-    public void gotPacket(PcapPacket pcapPacket) {
-        IpV4Packet ipPacket = pcapPacket.get(IpV4Packet.class);
-        TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class);
-
-        if (ipPacket == null || tcpPacket == null) {
-            return;
-        }
-        // ... TODO?
-        processPacket(pcapPacket);
-//        Class clazz = pcapPacket.getClass();
-//        RadiotapPacket radiotapPacket = pcapPacket.get(RadiotapPacket.class);
-//        Dot11ManagementPacket dot11ManagementPacket = pcapPacket.get(Dot11ManagementPacket.class);
-//        if (dot11ManagementPacket != null) {
-//            return;
-//        }
-//        if (radiotapPacket != null) {
-//            processRadiotapPacket(pcapPacket);
-//        }
-    }
-
-    /**
-     * Get the reassembled TCP connections. Note that if this is called while packets are still being processed (by
-     * calls to {@link #gotPacket(PcapPacket)}), the behavior is undefined and the returned list may be inconsistent.
-     * @return The reassembled TCP connections.
-     */
-    public List<Conversation> getTcpConversations() {
-        ArrayList<Conversation> combined = new ArrayList<>();
-        combined.addAll(mTerminatedConversations);
-        combined.addAll(mOpenConversations.values());
-        return combined;
-    }
-
-    private void processRadiotapPacket(PcapPacket pcapPacket) {
-        RadiotapPacket radiotapPacket = pcapPacket.get(RadiotapPacket.class);
-
-        RadiotapPacket.RadiotapHeader header = radiotapPacket.getHeader();
-        short length = header.getLength();
-        ArrayList<RadiotapPacket.RadiotapData> radiotapData = header.getDataFields();
-        // TODO: We can handle this 802.11 QoS data by creating our own class
-        // TODO: We only need to handle the first few bytes for source, destination, receiver, and transmitter
-        // TODO: addresses
-        Packet dataPacket = radiotapPacket.getPayload();
-        int dataLength = dataPacket.length();
-    }
-
-    private void processPacket(PcapPacket pcapPacket) {
-        TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class);
-        // Handle client connection initiation attempts.
-        if (tcpPacket.getHeader().getSyn() && !tcpPacket.getHeader().getAck()) {
-            // A segment with the SYN flag set, but no ACK flag indicates that a client is attempting to initiate a new
-            // connection.
-            processNewConnectionRequest(pcapPacket);
-            return;
-        }
-        // Handle server connection initiation acknowledgement
-        if (tcpPacket.getHeader().getSyn() && tcpPacket.getHeader().getAck()) {
-            // A segment with both the SYN and ACK flags set indicates that the server has accepted the client's request
-            // to initiate a new connection.
-            processNewConnectionAck(pcapPacket);
-            return;
-        }
-        // Handle resets
-        if (tcpPacket.getHeader().getRst()) {
-            processRstPacket(pcapPacket);
-            return;
-        }
-        // Handle FINs
-        if (tcpPacket.getHeader().getFin()) {
-            // Handle FIN packet.
-            processFinPacket(pcapPacket);
-        }
-        // Handle ACKs (currently only ACKs of FINS)
-        if (tcpPacket.getHeader().getAck()) {
-            processAck(pcapPacket);
-        }
-        // Handle packets that carry payload (application data).
-        if (tcpPacket.getPayload() != null) {
-            processPayloadPacket(pcapPacket);
-        }
-    }
-
-    private void processNewConnectionRequest(PcapPacket clientSynPacket) {
-        // A SYN w/o ACK always originates from the client.
-        Conversation conv = Conversation.fromPcapPacket(clientSynPacket, true);
-        conv.addSynPacket(clientSynPacket);
-        // Is there an ongoing conversation for the same four tuple (clientIp, clientPort, serverIp, serverPort) as
-        // found in the new SYN packet?
-        Conversation ongoingConv = mOpenConversations.get(conv);
-        if (ongoingConv != null) {
-            if (ongoingConv.isRetransmission(clientSynPacket)) {
-                // SYN retransmission detected, do nothing.
-                return;
-                // TODO: the way retransmission detection is implemented may cause a bug for connections where we have
-                // not recorded the initial SYN, but only the SYN ACK, as retransmission is determined by comparing the
-                // sequence numbers of initial SYNs -- and if no initial SYN is present for the Conversation, the new
-                // SYN will be interpreted as a retransmission. Possible fix: let isRentransmission ALWAYS return false
-                // when presented with a SYN packet when the Conversation already holds a SYN ACK packet?
-            } else {
-                // New SYN has different sequence number than SYN recorded for ongoingConv, so this must be an attempt
-                // to establish a new conversation with the same four tuple as ongoingConv.
-                // Mark existing connection as terminated.
-                // TODO: is this 100% theoretically correct, e.g., if many connection attempts are made back to back? And RST packets?
-                mTerminatedConversations.add(ongoingConv);
-                mOpenConversations.remove(ongoingConv);
-            }
-        }
-        // Finally, update the map of open connections with the new connection.
-        mOpenConversations.put(conv, conv);
-    }
-
-
-    /*
-     * TODO a problem across the board for all processXPacket methods below:
-     * if we start the capture in the middle of a TCP connection, we will not have an entry for the conversation in the
-     * map as we have not seen the initial SYN packet.
-     * Two ways we can address this:
-     * a) Perform null-checks and ignore packets for which we have not seen SYN
-     *    + easy to get correct
-     *    - we discard data (issue for long-lived connections!)
-     * b) Add a corresponding conversation entry whenever we encounter a packet that does not map to a conversation
-     *    + we consider all data
-     *    - not immediately clear if this will introduce bugs (incorrectly mapping packets to wrong conversations?)
-     *
-     *  [[[ I went with option b) for now; see getOngoingConversationOrCreateNew(PcapPacket pcapPacket). ]]]
-     */
-
-    private void processNewConnectionAck(PcapPacket srvSynPacket) {
-        // Find the corresponding ongoing connection, if any (if we start the capture just *after* the initial SYN, no
-        // ongoing conversation entry will exist, so it must be created in that case).
-//        Conversation conv = mOpenConversations.get(Conversation.fromPcapPacket(srvSynPacket, false));
-        Conversation conv = getOngoingConversationOrCreateNew(srvSynPacket);
-        // Note: exploits &&'s short-circuit operation: only attempts to add non-retransmissions.
-        if (!conv.isRetransmission(srvSynPacket) && !conv.addSynPacket(srvSynPacket)) {
-            // For safety/debugging: if NOT a retransmission and add fails,
-            // something has gone terribly wrong/invariant is broken.
-//            throw new AssertionError("Attempt to add SYN ACK packet that was NOT a retransmission failed." +
-//                    Conversation.class.getSimpleName() + " invariant broken.");
-        }
-    }
-
-    private void processRstPacket(PcapPacket rstPacket) {
-        Conversation conv = getOngoingConversationOrCreateNew(rstPacket);
-        // Add RST packet to conversation.
-        conv.addRstPacket(rstPacket);
-        // Move conversation to set of terminated conversations.
-        mTerminatedConversations.add(conv);
-        mOpenConversations.remove(conv, conv);
-    }
-
-    private void processFinPacket(PcapPacket finPacket) {
-//        getOngoingConversationForPacket(finPacket).addFinPacket(finPacket);
-        getOngoingConversationOrCreateNew(finPacket).addFinPacket(finPacket);
-    }
-
-    private void processAck(PcapPacket ackPacket) {
-//        getOngoingConversationForPacket(ackPacket).attemptAcknowledgementOfFin(ackPacket);
-        // Note that unlike the style for SYN, FIN, and payload packets, for "ACK only" packets, we want to avoid
-        // creating a new conversation.
-        Conversation conv = getOngoingConversationForPacket(ackPacket);
-        if (conv != null) {
-            // The ACK may be an ACK of a FIN, so attempt to mark the FIN as ack'ed.
-            conv.attemptAcknowledgementOfFin(ackPacket);
-            if (conv.isGracefullyShutdown()) {
-                // Move conversation to set of terminated conversations.
-                mTerminatedConversations.add(conv);
-                mOpenConversations.remove(conv);
-            }
-        }
-        // Note: add (additional) processing of ACKs (that are not ACKs of FINs) as necessary here...
-    }
-
-    private void processPayloadPacket(PcapPacket pcapPacket) {
-//        getOngoingConversationForPacket(pcapPacket).addPacket(pcapPacket, true);
-        getOngoingConversationOrCreateNew(pcapPacket).addPacket(pcapPacket, true);
-    }
-
-    /**
-     * Locates an ongoing conversation (if any) that {@code pcapPacket} pertains to.
-     * @param pcapPacket The packet that is to be mapped to an ongoing {@code Conversation}.
-     * @return The {@code Conversation} matching {@code pcapPacket} or {@code null} if there is no match.
-     */
-    private Conversation getOngoingConversationForPacket(PcapPacket pcapPacket) {
-        // We cannot know if this is a client-to-server or server-to-client packet without trying both options...
-        Conversation conv = mOpenConversations.get(Conversation.fromPcapPacket(pcapPacket, true));
-        if (conv == null) {
-            conv = mOpenConversations.get(Conversation.fromPcapPacket(pcapPacket, false));
-        }
-        return conv;
-    }
-
-    /**
-     * Like {@link #getOngoingConversationForPacket(PcapPacket)}, but creates and inserts a new {@code Conversation}
-     * into {@link #mOpenConversations} if no open conversation is found (i.e., in the case that
-     * {@link #getOngoingConversationForPacket(PcapPacket)} returns {@code null}).
-     *
-     * @param pcapPacket The packet that is to be mapped to an ongoing {@code Conversation}.
-     * @return The existing, ongoing {@code Conversation} matching {@code pcapPacket} or the newly created one in case
-     *         no match was found.
-     */
-    private Conversation getOngoingConversationOrCreateNew(PcapPacket pcapPacket) {
-        Conversation conv = getOngoingConversationForPacket(pcapPacket);
-        if (conv == null) {
-            TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class);
-            if (tcpPacket.getHeader().getSyn() && tcpPacket.getHeader().getAck()) {
-                // A SYN ACK packet always originates from the server (it is a reply to the initial SYN packet from the client)
-                conv = Conversation.fromPcapPacket(pcapPacket, false);
-            } else {
-                // TODO: can we do anything else but arbitrarily select who is designated as the server in this case?
-                // We can check if the IP prefix matches a local IP when handling traffic observed inside the local
-                // network, but that obviously won't be a useful strategy for an observer at the WAN port.
-                String srcIp = pcapPacket.get(IpV4Packet.class).getHeader().getSrcAddr().getHostAddress();
-                // TODO: REPLACE THE ROUTER'S IP WITH A PARAMETER!!!
-                boolean clientIsSrc = srcIp.startsWith("10.") || srcIp.startsWith("192.168.") || srcIp.equals("128.195.205.105");
-                conv = Conversation.fromPcapPacket(pcapPacket, clientIsSrc);
-            }
-            mOpenConversations.put(conv, conv);
-        }
-        return conv;
-    }
-}
index 7484d8aedc33b5573bdd53dc06610507081a585c..f1c264bf4dba24d454ececa0b68d50e85e83b7dd 100644 (file)
@@ -1,6 +1,6 @@
 package edu.uci.iotproject.analysis;
 
-import edu.uci.iotproject.Conversation;
+import edu.uci.iotproject.trafficreassembly.layer3.Conversation;
 import edu.uci.iotproject.DnsMap;
 import edu.uci.iotproject.util.PcapPacketUtils;
 import org.pcap4j.core.PcapPacket;
index ebc87b1a67db6cfb3be6c695750676ebb942207b..983de127902a7356c161b06db72716951f4f1450 100644 (file)
@@ -1,8 +1,8 @@
 package edu.uci.iotproject.analysis;
 
-import edu.uci.iotproject.Conversation;
+import edu.uci.iotproject.trafficreassembly.layer3.Conversation;
 import edu.uci.iotproject.DnsMap;
-import edu.uci.iotproject.TcpReassembler;
+import edu.uci.iotproject.trafficreassembly.layer3.TcpReassembler;
 import org.pcap4j.core.PacketListener;
 import org.pcap4j.core.PcapPacket;
 
index f3bd58519424205e12f44af554bf987ff5b04fa6..cadadbeaa4b4256be0ade9f5fe8f16868782a5cd 100644 (file)
@@ -1,6 +1,6 @@
 package edu.uci.iotproject.comparison;
 
-import edu.uci.iotproject.Conversation;
+import edu.uci.iotproject.trafficreassembly.layer3.Conversation;
 import edu.uci.iotproject.FlowPattern;
 
 /**
index 15eda20fe115e5b92b741292cb7fff4ad0a986f2..960d6a79e76d0fa70821a8bb92acb0fd5bf287bc 100644 (file)
@@ -1,6 +1,6 @@
 package edu.uci.iotproject.comparison;
 
-import edu.uci.iotproject.Conversation;
+import edu.uci.iotproject.trafficreassembly.layer3.Conversation;
 import edu.uci.iotproject.FlowPattern;
 import org.pcap4j.core.PcapPacket;
 import org.pcap4j.packet.TcpPacket;
index 3ddd884e940ece15627b471ec2a4e8e9b6b529bd..960a0d007147ba63abeaca6a404f3a721e254796 100644 (file)
@@ -1,6 +1,6 @@
 package edu.uci.iotproject.comparison;
 
-import edu.uci.iotproject.Conversation;
+import edu.uci.iotproject.trafficreassembly.layer3.Conversation;
 import edu.uci.iotproject.FlowPattern;
 
 /**
index 808dee73fab6647e84f7139e734a91a382332cc3..5a888b79a6e34bb731f60f86cf5728a432fa7fde 100644 (file)
@@ -1,6 +1,6 @@
 package edu.uci.iotproject.comparison;
 
-import edu.uci.iotproject.Conversation;
+import edu.uci.iotproject.trafficreassembly.layer3.Conversation;
 import edu.uci.iotproject.FlowPattern;
 
 import java.util.concurrent.Callable;
index 297107d66c10e7001dd7be8aa61d7aead8f83aef..2d193a9835f17818f36a2c74540a15ede51e2f4f 100644 (file)
@@ -1,6 +1,6 @@
 package edu.uci.iotproject.comparison.seqalignment;
 
-import edu.uci.iotproject.Conversation;
+import edu.uci.iotproject.trafficreassembly.layer3.Conversation;
 import org.pcap4j.core.PcapPacket;
 
 import java.util.List;
index c6119684cb270d192f8c1c4e6ba3135f5c3f6f2c..6aaa31854d51721d9b6d4edde0af878f8188ca8a 100644 (file)
@@ -1,9 +1,8 @@
 package edu.uci.iotproject.comparison.seqalignment;
 
-import edu.uci.iotproject.Conversation;
+import edu.uci.iotproject.trafficreassembly.layer3.Conversation;
 import edu.uci.iotproject.analysis.TcpConversationUtils;
 
-import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/AbstractClusterMatcher.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/AbstractClusterMatcher.java
new file mode 100644 (file)
index 0000000..bda6493
--- /dev/null
@@ -0,0 +1,49 @@
+package edu.uci.iotproject.detection;
+
+import org.pcap4j.core.PcapPacket;
+
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Base class for classes that search a traffic trace for sequences of packets that "belong to" a given cluster (in
+ * other words, classes that attempt to classify traffic as pertaining to a given cluster).
+ *
+ * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
+ * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
+ */
+abstract public class AbstractClusterMatcher {
+
+    /**
+     * The cluster that describes the sequence of packets that this {@link AbstractClusterMatcher} is trying to detect
+     * in the observed traffic.
+     */
+    protected final List<List<PcapPacket>> mCluster;
+
+    protected AbstractClusterMatcher(List<List<PcapPacket>> cluster) {
+        // ===================== PRECONDITION SECTION =====================
+        cluster = Objects.requireNonNull(cluster, "cluster cannot be null");
+        if (cluster.isEmpty() || cluster.stream().anyMatch(inner -> inner.isEmpty())) {
+            throw new IllegalArgumentException("cluster is empty (or contains an empty inner List)");
+        }
+        for (List<PcapPacket> clusterMember : cluster) {
+            if (clusterMember.size() != cluster.get(0).size()) {
+                throw new IllegalArgumentException("All sequences in cluster must contain the same number of packets");
+            }
+        }
+        // ================================================================
+        // Let the subclass prune the provided cluster
+        mCluster = pruneCluster(cluster);
+    }
+
+    /**
+     * Allows subclasses to specify how to prune the input cluster provided to the constructor.
+     * @param cluster The input cluster provided to the constructor.
+     * @return The pruned cluster to use in place of the input cluster.
+     */
+    abstract protected List<List<PcapPacket>> pruneCluster(List<List<PcapPacket>> cluster);
+
+    // TODO: move Direction outside Conversation so that this is less confusing.
+//    abstract protected Conversation.Direction[] getPacketDirections(List<PcapPacket> packets);
+
+}
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/ClusterMatcher.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/ClusterMatcher.java
deleted file mode 100644 (file)
index 33d6dbb..0000000
+++ /dev/null
@@ -1,362 +0,0 @@
-package edu.uci.iotproject.detection;
-
-import edu.uci.iotproject.Conversation;
-import edu.uci.iotproject.TcpReassembler;
-import edu.uci.iotproject.analysis.TcpConversationUtils;
-import edu.uci.iotproject.io.PcapHandleReader;
-import edu.uci.iotproject.util.PrintUtils;
-import org.pcap4j.core.*;
-
-import java.time.ZoneId;
-import java.util.*;
-import java.util.stream.Collectors;
-
-import static edu.uci.iotproject.util.PcapPacketUtils.*;
-
-/**
- * Searches a traffic trace for sequences of packets "belong to" a given cluster (in other words, attempts to classify
- * traffic as pertaining to a given cluster).
- *
- * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
- * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
- */
-public class ClusterMatcher implements PacketListener {
-
-    // Test client
-    public static void main(String[] args) throws PcapNativeException, NotOpenException {
-
-//        String path = "/scratch/July-2018"; // Rahmadi
-        String path = "/Users/varmarken/temp/UCI IoT Project/experiments"; // Janus
-        final String inputPcapFile = path + "/2018-07/dlink/dlink.wlan1.local.pcap";
-        final String signatureFile = path + "/2018-07/dlink/offSignature1.sig";
-
-        List<List<PcapPacket>> signature = PrintUtils.deserializeClustersFromFile(signatureFile);
-        ClusterMatcher clusterMatcher = new ClusterMatcher(signature, null,
-                (sig, match) -> System.out.println(
-                        String.format("[ !!! SIGNATURE DETECTED AT %s !!! ]",
-                                match.get(0).getTimestamp().atZone(ZoneId.of("America/Los_Angeles")))
-                )
-        );
-
-        PcapHandle handle;
-        try {
-            handle = Pcaps.openOffline(inputPcapFile, PcapHandle.TimestampPrecision.NANO);
-        } catch (PcapNativeException pne) {
-            handle = Pcaps.openOffline(inputPcapFile);
-        }
-        PcapHandleReader reader = new PcapHandleReader(handle, p -> true, clusterMatcher);
-        reader.readFromHandle();
-        clusterMatcher.performDetection();
-    }
-
-    /**
-     * The cluster that describes the sequence of packets that this {@link ClusterMatcher} is trying to detect in the
-     * observed traffic.
-     */
-    private final List<List<PcapPacket>> mCluster;
-
-    /**
-     * The ordered directions of packets in the sequences that make up {@link #mCluster}.
-     */
-    private final Conversation.Direction[] mClusterMemberDirections;
-
-    /**
-     * For reassembling the observed traffic into TCP connections.
-     */
-    private final TcpReassembler mTcpReassembler = new TcpReassembler();
-
-    /**
-     * IP of the router's WAN port (if analyzed traffic is captured at the ISP's point of view).
-     */
-    private final String mRouterWanIp;
-
-    private final ClusterMatchObserver[] mObservers;
-
-    /**
-     * Create a {@link ClusterMatcher}.
-     * @param cluster The cluster that traffic is matched against.
-     * @param routerWanIp The router's WAN IP if examining traffic captured at the ISP's point of view (used for
-     *                    determining the direction of packets).
-     * @param detectionObservers Client code that wants to get notified whenever the {@link ClusterMatcher} detects that
-     *                          (a subset of) the examined traffic is similar to the traffic that makes up
-     *                          {@code cluster}, i.e., when the examined traffic is classified as pertaining to
-     *                          {@code cluster}.
-     */
-    public ClusterMatcher(List<List<PcapPacket>> cluster, String routerWanIp, ClusterMatchObserver... detectionObservers) {
-        // ===================== PRECONDITION SECTION =====================
-        cluster = Objects.requireNonNull(cluster, "cluster cannot be null");
-        if (cluster.isEmpty() || cluster.stream().anyMatch(inner -> inner.isEmpty())) {
-            throw new IllegalArgumentException("cluster is empty (or contains an empty inner List)");
-        }
-        mObservers = Objects.requireNonNull(detectionObservers, "detectionObservers cannot be null");
-        if (mObservers.length == 0) {
-            throw new IllegalArgumentException("no detectionObservers provided");
-        }
-        // Build the cluster members' direction sequence.
-        // Note: assumes that the provided cluster was captured within the local network (routerWanIp is set to null).
-        mClusterMemberDirections = getPacketDirections(cluster.get(0), null);
-        /*
-         * Enforce restriction on cluster members: all representatives must exhibit the same direction pattern and
-         * contain the same number of packets. Note that this is a somewhat heavy operation, so it may be disabled later
-         * on in favor of performance. However, it is only run once (at instantiation), so the overhead may be warranted
-         * in order to ensure correctness, especially during the development/debugging phase.
-         */
-        if (cluster.stream().
-                anyMatch(inner -> !Arrays.equals(mClusterMemberDirections, getPacketDirections(inner, null)))) {
-            throw new IllegalArgumentException(
-                    "cluster members must contain the same number of packets and exhibit the same packet direction " +
-                            "pattern"
-            );
-        }
-        // ================================================================
-        // Prune the provided cluster.
-        mCluster = pruneCluster(cluster);
-        mRouterWanIp = routerWanIp;
-    }
-
-    @Override
-    public void gotPacket(PcapPacket packet) {
-        // Present packet to TCP reassembler so that it can be mapped to a connection (if it is a TCP packet).
-        mTcpReassembler.gotPacket(packet);
-    }
-
-    /**
-     * Get the cluster that describes the packet sequence that this {@link ClusterMatcher} is searching for.
-     * @return the cluster that describes the packet sequence that this {@link ClusterMatcher} is searching for.
-     */
-    public List<List<PcapPacket>> getCluster() {
-        return mCluster;
-    }
-
-    public void performDetection() {
-        /*
-         * Let's start out simple by building a version that only works for signatures that do not span across multiple
-         * TCP conversations...
-         */
-        for (Conversation c : mTcpReassembler.getTcpConversations()) {
-            if (c.isTls() && c.getTlsApplicationDataPackets().isEmpty() || !c.isTls() && c.getPackets().isEmpty()) {
-                // Skip empty conversations.
-                continue;
-            }
-            for (List<PcapPacket> signatureSequence : mCluster) {
-                if (isTlsSequence(signatureSequence) != c.isTls()) {
-                    // We consider it a mismatch if one is a TLS application data sequence and the other is not.
-                    continue;
-                }
-                // Fetch set of packets to examine based on TLS or not.
-                List<PcapPacket> cPkts = c.isTls() ? c.getTlsApplicationDataPackets() : c.getPackets();
-                /*
-                 * Note: we embed the attempt to detect the signature sequence in a loop in order to capture those cases
-                 * where the same signature sequence appears multiple times in one Conversation.
-                 *
-                 * Note: since we expect all sequences that together make up the signature to exhibit the same direction
-                 * pattern, we can simply pass the precomputed direction array for the signature sequence so that it
-                 * won't have to be recomputed internally in each call to findSubsequenceInSequence().
-                 */
-                Optional<List<PcapPacket>> match;
-                while ((match = findSubsequenceInSequence(signatureSequence, cPkts, mClusterMemberDirections, null)).
-                        isPresent()) {
-                    List<PcapPacket> matchSeq = match.get();
-                    // Notify observers about the match.
-                    Arrays.stream(mObservers).forEach(o -> o.onMatch(ClusterMatcher.this, matchSeq));
-                    /*
-                     * Get the index in cPkts of the last packet in the sequence of packets that matches the searched
-                     * signature sequence.
-                     */
-                    int matchSeqEndIdx = cPkts.indexOf(matchSeq.get(matchSeq.size()-1));
-                    // We restart the search for the signature sequence immediately after that index, so truncate cPkts.
-                    cPkts = cPkts.stream().skip(matchSeqEndIdx + 1).collect(Collectors.toList());
-                }
-            }
-            /*
-             * TODO:
-             * if no item in cluster matches, also perform a distance-based matching to cover those cases where we did
-             * not manage to capture every single mutation of the sequence during training.
-             *
-             * Need to compute average/centroid of cluster to do so...? Compute within-cluster variance, then check if
-             * distance between input conversation and cluster average/centroid is smaller than or equal to the computed
-             * variance?
-             */
-        }
-    }
-
-    /**
-     * Checks if {@code sequence} is a sequence of TLS packets. Note: the current implementation relies on inspection
-     * of the port numbers when deciding between TLS vs. non-TLS. Therefore, only the first packet of {@code sequence}
-     * is examined as it is assumed that all packets in {@code sequence} pertain to the same {@link Conversation} and
-     * hence share the same set of two src/dst port numbers (albeit possibly alternating between which one is the src
-     * and which one is the dst, as packets in {@code sequence} may be in alternating directions).
-     * @param sequence The sequence of packets for which it is to be determined if it is a sequence of TLS packets or
-     *                 non-TLS packets.
-     * @return {@code true} if {@code sequence} is a sequence of TLS packets, {@code false} otherwise.
-     */
-    private boolean isTlsSequence(List<PcapPacket> sequence) {
-        // NOTE: Assumes ALL packets in sequence pertain to the same TCP connection!
-        PcapPacket firstPkt = sequence.get(0);
-        int srcPort = getSourcePort(firstPkt);
-        int dstPort = getDestinationPort(firstPkt);
-        return TcpConversationUtils.isTlsPort(srcPort) || TcpConversationUtils.isTlsPort(dstPort);
-    }
-
-    /**
-     * Examine if a given sequence of packets ({@code sequence}) contains a given shorter sequence of packets
-     * ({@code subsequence}). Note: the current implementation actually searches for a substring as it does not allow
-     * for interleaving packets in {@code sequence} that are not in {@code subsequence}; for example, if
-     * {@code subsequence} consists of packet lengths [2, 3, 5] and {@code sequence} consists of  packet lengths
-     * [2, 3, 4, 5], the result will be that there is no match (because of the interleaving 4). If we are to allow
-     * interleaving packets, we need a modified version of
-     * <a href="https://stackoverflow.com/a/20545604/1214974">this</a>.
-     *
-     * @param subsequence The sequence to search for.
-     * @param sequence The sequence to search.
-     * @param subsequenceDirections The directions of packets in {@code subsequence} such that for all {@code i},
-     *                              {@code subsequenceDirections[i]} is the direction of the packet returned by
-     *                              {@code subsequence.get(i)}. May be set to {@code null}, in which this call will
-     *                              internally compute the packet directions.
-     * @param sequenceDirections The directions of packets in {@code sequence} such that for all {@code i},
-     *                           {@code sequenceDirections[i]} is the direction of the packet returned by
-     *                           {@code sequence.get(i)}. May be set to {@code null}, in which this call will internally
-     *                           compute the packet directions.
-     *
-     * @return An {@link Optional} containing the part of {@code sequence} that matches {@code subsequence}, or an empty
-     *         {@link Optional} if no part of {@code sequence} matches {@code subsequence}.
-     */
-    private Optional<List<PcapPacket>> findSubsequenceInSequence(List<PcapPacket> subsequence,
-                                                                 List<PcapPacket> sequence,
-                                                                 Conversation.Direction[] subsequenceDirections,
-                                                                 Conversation.Direction[] sequenceDirections) {
-        if (sequence.size() < subsequence.size()) {
-            // If subsequence is longer, it cannot be contained in sequence.
-            return Optional.empty();
-        }
-        if (isTlsSequence(subsequence) != isTlsSequence(sequence)) {
-            // We consider it a mismatch if one is a TLS application data sequence and the other is not.
-            return Optional.empty();
-        }
-        // If packet directions have not been precomputed by calling code, we need to construct them.
-        if (subsequenceDirections == null) {
-            subsequenceDirections = getPacketDirections(subsequence, mRouterWanIp);
-        }
-        if (sequenceDirections == null) {
-            sequenceDirections = getPacketDirections(sequence, mRouterWanIp);
-        }
-        int subseqIdx = 0;
-        int seqIdx = 0;
-        while (seqIdx < sequence.size()) {
-            PcapPacket subseqPkt = subsequence.get(subseqIdx);
-            PcapPacket seqPkt = sequence.get(seqIdx);
-            // We only have a match if packet lengths and directions match.
-            if (subseqPkt.getOriginalLength() == seqPkt.getOriginalLength() &&
-                    subsequenceDirections[subseqIdx] == sequenceDirections[seqIdx]) {
-                // A match; advance both indices to consider next packet in subsequence vs. next packet in sequence.
-                subseqIdx++;
-                seqIdx++;
-                if (subseqIdx == subsequence.size()) {
-                    // We managed to match the entire subsequence in sequence.
-                    // Return the sublist of sequence that matches subsequence.
-                    /*
-                     * TODO:
-                     * ASSUMES THE BACKING LIST (i.e., 'sequence') IS _NOT_ STRUCTURALLY MODIFIED, hence may not work
-                     * for live traces!
-                     */
-                    return Optional.of(sequence.subList(seqIdx - subsequence.size(), seqIdx));
-                }
-            } else {
-                // Mismatch.
-                if (subseqIdx > 0) {
-                    /*
-                     * If we managed to match parts of subsequence, we restart the search for subsequence in sequence at
-                     * the index of sequence where the current mismatch occurred. I.e., we must reset subseqIdx, but
-                     * leave seqIdx untouched.
-                     */
-                    subseqIdx = 0;
-                } else {
-                    /*
-                     * First packet of subsequence didn't match packet at seqIdx of sequence, so we move forward in
-                     * sequence, i.e., we continue the search for subsequence in sequence starting at index seqIdx+1 of
-                     * sequence.
-                     */
-                    seqIdx++;
-                }
-            }
-        }
-        return Optional.empty();
-    }
-
-    /**
-     * Given a cluster, produces a pruned version of that cluster. In the pruned version, there are no duplicate cluster
-     * members. Two cluster members are considered identical if their packets lengths and packet directions are
-     * identical. The resulting pruned cluster is unmodifiable (this applies to both the outermost list as well as the
-     * nested lists) in order to preserve its integrity when exposed to external code (e.g., through
-     * {@link #getCluster()}).
-     *
-     * @param cluster A cluster to prune.
-     * @return The resulting pruned cluster.
-     */
-    private final List<List<PcapPacket>> pruneCluster(List<List<PcapPacket>> cluster) {
-        List<List<PcapPacket>> prunedCluster = new ArrayList<>();
-        for (List<PcapPacket> originalClusterSeq : cluster) {
-            boolean alreadyPresent = false;
-            for (List<PcapPacket> prunedClusterSeq : prunedCluster) {
-                Optional<List<PcapPacket>> duplicate = findSubsequenceInSequence(originalClusterSeq, prunedClusterSeq,
-                        mClusterMemberDirections, mClusterMemberDirections);
-                if (duplicate.isPresent()) {
-                    alreadyPresent = true;
-                    break;
-                }
-            }
-            if (!alreadyPresent) {
-                prunedCluster.add(Collections.unmodifiableList(originalClusterSeq));
-            }
-        }
-        return Collections.unmodifiableList(prunedCluster);
-    }
-
-    /**
-     * Given a {@code List<PcapPacket>}, generate a {@code Conversation.Direction[]} such that each entry in the
-     * resulting {@code Conversation.Direction[]} specifies the direction of the {@link PcapPacket} at the corresponding
-     * index in the input list.
-     * @param packets The list of packets for which to construct a corresponding array of packet directions.
-     * @param routerWanIp The IP of the router's WAN port. This is used for determining the direction of packets when
-     *                    the traffic is captured just outside the local network (at the ISP side of the router). Set to
-     *                    {@code null} if {@code packets} stem from traffic captured within the local network.
-     * @return A {@code Conversation.Direction[]} specifying the direction of the {@link PcapPacket} at the
-     *         corresponding index in {@code packets}.
-     */
-    private static Conversation.Direction[] getPacketDirections(List<PcapPacket> packets, String routerWanIp) {
-        Conversation.Direction[] directions = new Conversation.Direction[packets.size()];
-        for (int i = 0; i < packets.size(); i++) {
-            PcapPacket pkt = packets.get(i);
-            if (getSourceIp(pkt).equals(getDestinationIp(pkt))) {
-                // Sanity check: we shouldn't be processing loopback traffic
-                throw new AssertionError("loopback traffic detected");
-            }
-            if (isSrcIpLocal(pkt) || getSourceIp(pkt).equals(routerWanIp)) {
-                directions[i] = Conversation.Direction.CLIENT_TO_SERVER;
-            } else if (isDstIpLocal(pkt) || getDestinationIp(pkt).equals(routerWanIp)) {
-                directions[i] = Conversation.Direction.SERVER_TO_CLIENT;
-            } else {
-                //throw new IllegalArgumentException("no local IP or router WAN port IP found, can't detect direction");
-            }
-        }
-        return directions;
-    }
-
-    /**
-     * Interface used by client code to register for receiving a notification whenever the {@link ClusterMatcher}
-     * detects traffic that is similar to the traffic that makes up the cluster returned by
-     * {@link ClusterMatcher#getCluster()}.
-     */
-    interface ClusterMatchObserver {
-        /**
-         * Callback that is invoked whenever a sequence that is similar to a sequence associated with the cluster (i.e.,
-         * a sequence is a member of the cluster) is detected in the traffic that the associated {@link ClusterMatcher}
-         * observes.
-         * @param clusterMatcher The {@link ClusterMatcher} that detected a match (classified traffic as pertaining to
-         *                       its associated cluster).
-         * @param match The traffic that was deemed to match the cluster associated with {@code clusterMatcher}.
-         */
-        void onMatch(ClusterMatcher clusterMatcher, List<PcapPacket> match);
-    }
-
-}
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/SignatureDetector.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/SignatureDetector.java
deleted file mode 100644 (file)
index 9d8b4d2..0000000
+++ /dev/null
@@ -1,662 +0,0 @@
-package edu.uci.iotproject.detection;
-
-import edu.uci.iotproject.analysis.TriggerTrafficExtractor;
-import edu.uci.iotproject.analysis.UserAction;
-import edu.uci.iotproject.io.PcapHandleReader;
-import edu.uci.iotproject.util.PrintUtils;
-import org.jgrapht.GraphPath;
-import org.jgrapht.alg.shortestpath.DijkstraShortestPath;
-import org.jgrapht.graph.DefaultWeightedEdge;
-import org.jgrapht.graph.SimpleDirectedWeightedGraph;
-import org.pcap4j.core.*;
-
-import java.time.Duration;
-import java.time.Instant;
-import java.time.ZoneId;
-import java.time.format.DateTimeFormatter;
-import java.time.format.FormatStyle;
-import java.util.*;
-import java.util.function.Consumer;
-import java.util.stream.Collectors;
-
-/**
- * Detects an event signature that spans one or multiple TCP connections.
- *
- * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
- * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
- */
-public class SignatureDetector implements PacketListener, ClusterMatcher.ClusterMatchObserver {
-
-    // Test client
-    public static void main(String[] args) throws PcapNativeException, NotOpenException {
-//        if (args.length < 3) {
-//            String errMsg = String.format("Usage: %s inputPcapFile onSignatureFile offSignatureFile",
-//                    SignatureDetector.class.getSimpleName());
-//            System.out.println(errMsg);
-//            return;
-//        }
-//        final String inputPcapFile = args[0];
-//        final String onSignatureFile = args[1];
-//        final String offSignatureFile = args[2];
-
-        String path = "/scratch/July-2018"; // Rahmadi
-//        String path = "/Users/varmarken/temp/UCI IoT Project/experiments"; // Janus
-//        String path = "/home/jvarmark/iot_project/datasets"; // Hera (server)
-//        String path = "/raid/varmarken/iot_project/datasets"; // Zeus (server)
-
-        // No activity test
-        //final String inputPcapFile = path + "/evaluation/no-activity/no-activity.wlan1.pcap";
-
-        // D-Link Siren experiment
-//        final String inputPcapFile = path + "/evaluation/dlink-siren/dlink-siren.data.wlan1.pcap";
-//        final String inputPcapFile = path + "/evaluation/dlink-siren/dlink-siren.eth0.local.pcap";
-        // D-Link Siren DEVICE signatures
-//        final String onSignatureFile = path + "/2018-08/dlink-siren/onSignature-DLink-Siren-device.sig";
-//        final String offSignatureFile = path + "/2018-08/dlink-siren/offSignature-DLink-Siren-device.sig";
-        // D-Link Siren PHONE signatures
-//        final String onSignatureFile = path + "/2018-08/dlink-siren/onSignature-DLink-Siren-phone.sig";
-//        final String offSignatureFile = path + "/2018-08/dlink-siren/offSignature-DLink-Siren-phone.sig";
-        // TODO: EXPERIMENT - November 19, 2018
-        // Hue Bulb experiment
-//        final String inputPcapFile = path + "/2018-08/hue-bulb/hue-bulb.wlan1.local.pcap";
-        // Hue Bulb PHONE signatures
-//        final String onSignatureFile = path + "/experimental_result/standalone/hue-bulb/signatures/hue-bulb-onSignature-phone-side.sig";
-//        final String offSignatureFile = path + "/experimental_result/standalone/hue-bulb/signatures/hue-bulb-offSignature-phone-side.sig";
-
-        /*
-        // Kwikset Doorlock Sep 12 experiment
-//        final String inputPcapFile = path + "/evaluation/kwikset-doorlock/kwikset-doorlock.data.wlan1.pcap";
-        final String inputPcapFile = path + "/evaluation/kwikset-doorlock/kwikset-doorlock.data.eth0.pcap";
-//        // Kwikset Doorlock PHONE signatures
-        final String onSignatureFile = path + "/2018-08/kwikset-doorlock/onSignature-Kwikset-Doorlock-phone-new.sig";
-        final String offSignatureFile = path + "/2018-08/kwikset-doorlock/offSignature-Kwikset-Doorlock-phone-new.sig";
-        */
-
-        // D-Link Plug experiment
-        //final String inputPcapFile = path + "/evaluation/dlink/dlink-plug.data.wlan1.pcap";
-//        final String inputPcapFile = path + "/evaluation/dlink/dlink-plug.data.eth0.pcap";
-
-        // D-Link Plug DEVICE signatures
-//        final String onSignatureFile = path + "/2018-07/dlink/onSignature-DLink-Plug-device.sig";
-//        final String offSignatureFile = path + "/2018-07/dlink/offSignature-DLink-Plug-device.sig";
-        // D-Link Plug PHONE signatures
-//        final String onSignatureFile = path + "/2018-07/dlink/onSignature-DLink-Plug-phone.sig";
-//        final String offSignatureFile = path + "/2018-07/dlink/offSignature-DLink-Plug-phone.sig";
-
-        // TODO: The following are negative tests against the PCAP file from UNSW
-//        final String inputPcapFile = path + "/UNSW/16-10-04.pcap"; // TODO: Seems to be broken! Zero-payload!
-//        final String inputPcapFile = path + "/UNSW/16-10-12.pcap";
-
-//        final String inputPcapFile = path + "/UNSW/16-09-28.pcap"; // TODO: Seems to be broken! Zero-payload!
-//        final String inputPcapFile = path + "/UNSW/16-10-02.pcap"; // TODO: Seems to be broken!
-//        final String inputPcapFile = path + "/UNSW/16-10-03.pcap"; // TODO: Seems to be broken!
-//        final String inputPcapFile = path + "/UNSW/16-10-04-a.pcap"; // TODO: Seems to be broken! Zero-payload!
-//        final String inputPcapFile = path + "/UNSW/16-10-04-b.pcap"; // TODO: Seems to be broken! Zero-payload!
-//        final String inputPcapFile = path + "/UNSW/16-10-07.pcap"; // TODO: Seems to be broken!
-//        final String inputPcapFile = path + "/UNSW/16-10-08.pcap"; // TODO: Seems to be broken!
-//        final String inputPcapFile = path + "/UNSW/16-10-09.pcap"; // TODO: Seems to be broken!
-//        final String inputPcapFile = path + "/UNSW/16-10-10.pcap"; // TODO: Seems to be broken!
-//        final String inputPcapFile = path + "/UNSW/16-10-11.pcap"; // TODO: Seems to be broken!
-        // TODO: The following one is very long!!! - Split into smaller files!
-//        final String inputPcapFile = path + "/UNSW/16-10-12-a.pcap";
-//        final String inputPcapFile = path + "/UNSW/16-10-12-b.pcap";
-//        final String inputPcapFile = path + "/UNSW/16-10-12-c.pcap";
-//        final String inputPcapFile = path + "/UNSW/16-10-12-d.pcap";
-
-//        final String inputPcapFile = path + "/UNSW/16-09-23.pcap";
-//        final String inputPcapFile = path + "/UNSW/16-09-24.pcap";
-//        final String inputPcapFile = path + "/UNSW/16-09-25.pcap";
-//        final String inputPcapFile = path + "/UNSW/16-09-26.pcap";
-//        final String inputPcapFile = path + "/UNSW/16-09-27.pcap";
-//        final String inputPcapFile = path + "/UNSW/16-09-29.pcap";
-//        final String inputPcapFile = path + "/UNSW/16-10-01.pcap";
-//        final String inputPcapFile = path + "/UNSW/16-10-06.pcap";
-        // Negative test: dataset from UNB
-//        final String inputPcapFile = path + "/evaluation/negative-datasets/UNB/Monday-WorkingHours_one-local-endpoint.pcap";
-
-        // TODO: The following are tests for signatures against training data
-
-        // D-Link Plug experiment
-//        final String inputPcapFile = path + "/training/dlink-plug/wlan1/dlink-plug.wlan1.local.pcap";
-        // D-Link Plug DEVICE signatures
-//        final String onSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-onSignature-device-side.sig";
-//        final String offSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-offSignature-device-side.sig";
-        // D-Link Plug PHONE signatures
-//        final String onSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-onSignature-phone-side.sig";
-//        final String offSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-offSignature-phone-side.sig";
-
-        // TODO: EXPERIMENT - November 7, 2018
-        // D-Link Plug experiment
-        //final String inputPcapFile = path + "/experimental_result/standalone/dlink-plug/wlan1/dlink-plug.wlan1.local.pcap";
-        //final String inputPcapFile = path + "/experimental_result/smarthome/dlink-plug/wlan1/dlink-plug.wlan1.detection.pcap";
-        //final String inputPcapFile = path + "/experimental_result/smarthome/dlink-plug/eth0/dlink-plug.eth0.detection.pcap";
-        // D-Link Plug DEVICE signatures
-//        final String onSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-onSignature-device-side.sig";
-//        final String offSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-offSignature-device-side.sig";
-        // D-Link Plug PHONE signatures
-//        final String onSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-onSignature-phone-side.sig";
-//        final String offSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-offSignature-phone-side.sig";
-
-        // TODO: EXPERIMENT - November 9, 2018
-        // D-Link Siren experiment
-        //final String inputPcapFile = path + "/experimental_result/standalone/dlink-siren/wlan1/dlink-siren.wlan1.local.pcap";
-        //final String inputPcapFile = path + "/experimental_result/smarthome/dlink-siren/wlan1/dlink-siren.wlan1.detection.pcap";
-//        final String inputPcapFile = path + "/experimental_result/smarthome/dlink-siren/eth0/dlink-siren.eth0.detection.pcap";
-        // D-Link Siren DEVICE signatures
-        // TODO: The device signature does not have pairs---only one packet which is 216, so we don't consider this as a signature
-//        final String onSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-onSignature-device-side.sig";
-//        final String offSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-offSignature-device-side.sig";
-        // D-Link Siren PHONE signatures
-//        final String onSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-onSignature-phone-side.sig";
-//        final String offSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-offSignature-phone-side.sig";
-//        final String onSignatureFile = path + "/training/signatures/dlink-siren/dlink-siren-onSignature-phone-side.sig";
-//        final String offSignatureFile = path + "/training/signatures/dlink-siren/dlink-siren-offSignature-phone-side.sig";
-
-        // TP-Link Plug experiment
-////        final String inputPcapFile = path + "/training/tplink-plug/wlan1/tplink-plug.wlan1.local.pcap";
-////        final String inputPcapFile = path + "/experimental_result/wifi-Sniffer/tests2/airtool_2019-01-04_11.08.45.AM.pcap";
-//        final String inputPcapFile = path + "/experimental_result/wifi-Sniffer/tests2/command-frames-only.pcap";
-//        // TP-Link Plug DEVICE signatures
-//        final String onSignatureFile = path + "/training/tplink-plug/signatures/tplink-plug-onSignature-device-side.sig";
-//        final String offSignatureFile = path + "/training/tplink-plug/signatures/tplink-plug-offSignature-device-side.sig";
-        // TODO: EXPERIMENT - November 8, 2018
-        // TP-Link Plug experiment
-//        final String inputPcapFile = path + "/experimental_result/standalone/tplink-plug/wlan1/tplink-plug.wlan1.local.pcap";
-//        final String inputPcapFile = path + "/experimental_result/standalone/tplink-plug/eth0/tplink-plug.eth0.local.pcap";
-//        final String inputPcapFile = path + "/experimental_result/smarthome/tplink-plug/wlan1/tplink-plug.wlan1.detection.pcap";
-        //final String inputPcapFile = path + "/experimental_result/smarthome/tplink-plug/eth0/tplink-plug.eth0.detection.pcap";
-        // TP-Link Plug DEVICE signatures
-//        final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-device-side.sig";
-//        final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-device-side.sig";
-//        final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-device-side-outbound.sig";
-//        final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-device-side-outbound.sig";
-        // TP-Link Plug PHONE signatures
-//        final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-phone-side.sig";
-//        final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-phone-side.sig";
-
-        // Arlo camera experiment
-//        final String inputPcapFile = path + "/training/arlo-camera/wlan1/arlo-camera.wlan1.local.pcap";
-////        // TP-Link Plug DEVICE signatures
-//        final String onSignatureFile = path + "/training/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig";
-//        final String offSignatureFile = path + "/training/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig";
-        // TODO: EXPERIMENT - November 13, 2018
-        // Arlo Camera experiment
-//        final String inputPcapFile = path + "/experimental_result/standalone/arlo-camera/wlan1/arlo-camera.wlan1.local.pcap";
-//        final String inputPcapFile = path + "/experimental_result/standalone/arlo-camera/eth0/arlo-camera.eth0.local.pcap";
-//        final String inputPcapFile = path + "/experimental_result/smarthome/arlo-camera/wlan1/arlo-camera.wlan1.detection.pcap";
-//        final String inputPcapFile = path + "/experimental_result/smarthome/arlo-camera/eth0/arlo-camera.eth0.detection.pcap";
-//        final String inputPcapFile = path + "/training/arlo-camera/eth0/arlo-camera.eth0.local.pcap";
-        // Arlo Camera PHONE signatures
-//        final String onSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig";
-//        final String offSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig";
-
-        // Amazon Alexa experiment
-//        final String inputPcapFile = path + "/training/amazon-alexa/wlan1/alexa2.wlan1.local.pcap";
-//        // TP-Link Plug DEVICE signatures
-//        final String onSignatureFile = path + "/training/amazon-alexa/signatures/amazon-alexa-onSignature-device-side.sig";
-//        final String offSignatureFile = path + "/training/amazon-alexa/signatures/amazon-alexa-offSignature-device-side.sig";
-
-        // SmartThings Plug experiment
-//        final String inputPcapFile = path + "/training/st-plug/wlan1/st-plug.wlan1.local.pcap";
-//        // SmartThings Plug DEVICE signatures
-//        //final String onSignatureFile = path + "/training/st-plug/signatures/st-plug-onSignature-device-side.sig";
-//        //final String offSignatureFile = path + "/training/st-plug/signatures/st-plug-offSignature-device-side.sig";
-//        // SmartThings Plug PHONE signatures
-//        final String onSignatureFile = path + "/training/st-plug/signatures/st-plug-onSignature-phone-side.sig";
-//        final String offSignatureFile = path + "/training/st-plug/signatures/st-plug-offSignature-phone-side.sig";
-        // TODO: EXPERIMENT - November 12, 2018
-        // SmartThings Plug experiment
-//        final String inputPcapFile = path + "/experimental_result/standalone/st-plug/wlan1/st-plug.wlan1.local.pcap";
-//        final String inputPcapFile = path + "/experimental_result/standalone/st-plug/eth0/st-plug.eth0.local.pcap";
-//        //final String inputPcapFile = path + "/experimental_result/smarthome/st-plug/wlan1/st-plug.wlan1.detection.pcap";
-//        final String inputPcapFile = path + "/experimental_result/smarthome/st-plug/eth0/st-plug.eth0.detection.pcap";
-//        // SmartThings Plug PHONE signatures
-//        final String onSignatureFile = path + "/experimental_result/standalone/st-plug/signatures/st-plug-onSignature-phone-side.sig";
-//        final String offSignatureFile = path + "/experimental_result/standalone/st-plug/signatures/st-plug-offSignature-phone-side.sig";
-//        final String onSignatureFile = path + "/training/signatures/st-plug/st-plug-onSignature-phone-side.sig";
-//        final String offSignatureFile = path + "/training/signatures/st-plug/st-plug-offSignature-phone-side.sig";
-
-        // TODO: EXPERIMENT - January 9, 2018
-        // Blossom Sprinkler experiment
-//        final String inputPcapFile = path + "/experimental_result/standalone/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.local.pcap";
-        final String inputPcapFile = path + "/experimental_result/smarthome/blossom-sprinkler/eth0/blossom-sprinkler.eth0.detection.pcap";
-//        final String inputPcapFile = path + "/experimental_result/smarthome/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.detection.pcap";
-        // Blossom Sprinkler DEVICE signatures
-        final String onSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig";
-        final String offSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig";
-        // Blossom Sprinkler PHONE signatures
-//        final String onSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-phone-side.sig";
-//        final String offSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-phone-side.sig";
-
-        // LiFX Bulb experiment
-//        final String inputPcapFile = path + "/training/lifx-bulb/wlan1/lifx-bulb.wlan1.local.pcap";
-//        // LiFX Bulb DEVICE signatures
-//        final String onSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-onSignature-device-side.sig";
-//        final String offSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-offSignature-device-side.sig";
-        // LiFX Bulb PHONE signatures
-//        final String onSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-onSignature-phone-side.sig";
-//        final String offSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-offSignature-phone-side.sig";
-
-        // Blossom Sprinkler experiment
-//        //final String inputPcapFile = path + "/training/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.local.pcap";
-//        final String inputPcapFile = path + "/2018-08/blossom/blossom.wlan1.local.pcap";
-//        //final String inputPcapFile = path + "/training/blossom-sprinkler/eth0/blossom-sprinkler.eth0.local.pcap";
-//        // Blossom Sprinkler DEVICE signatures
-//        final String onSignatureFile = path + "/training/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig";
-//        final String offSignatureFile = path + "/training/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig";
-
-        // Nest Thermostat experiment
-//        final String inputPcapFile = path + "/training/nest-thermostat/wlan1/nest-thermostat.wlan1.local.pcap";
-//        // Nest Thermostat DEVICE signatures
-////        final String onSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-onSignature-device-side.sig";
-////        final String offSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-offSignature-device-side.sig";
-//        // Nest Thermostat PHONE signatures
-//        final String onSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-onSignature-phone-side.sig";
-//        final String offSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-offSignature-phone-side.sig";
-        // TODO: EXPERIMENT - November 15, 2018
-        // Nest Thermostat experiment
-//        final String inputPcapFile = path + "/experimental_result/standalone/nest-thermostat/wlan1/nest-thermostat.wlan1.local.pcap";
-////        final String inputPcapFile = path + "/experimental_result/standalone/nest-thermostat/eth0/nest-thermostat.eth0.local.pcap";
-////        final String inputPcapFile = path + "/experimental_result/smarthome/nest-thermostat/wlan1/nest-thermostat.wlan1.detection.pcap";
-//        final String inputPcapFile = path + "/experimental_result/smarthome/nest-thermostat/eth0/nest-thermostat.eth0.detection.pcap";
-////        // Nest Thermostat PHONE signatures
-//        final String onSignatureFile = path + "/experimental_result/standalone/nest-thermostat/signatures/nest-thermostat-onSignature-phone-side.sig";
-//        final String offSignatureFile = path + "/experimental_result/standalone/nest-thermostat/signatures/nest-thermostat-offSignature-phone-side.sig";
-
-        /*
-        // Hue Bulb experiment
-        final String inputPcapFile = path + "/training/hue-bulb/wlan1/hue-bulb.wlan1.local.pcap";
-        // Hue Bulb PHONE signatures
-        final String onSignatureFile = path + "/training/hue-bulb/signatures/hue-bulb-onSignature-phone-side.sig";
-        final String offSignatureFile = path + "/training/hue-bulb/signatures/hue-bulb-offSignature-phone-side.sig";
-        */
-
-
-
-        // TP-Link Bulb experiment
-//        final String inputPcapFile = path + "/training/tplink-bulb/wlan1/tplink-bulb.wlan1.local.pcap";
-//        // TP-Link Bulb PHONE signatures
-//        final String onSignatureFile = path + "/training/tplink-bulb/signatures/tplink-bulb-onSignature-phone-side.sig";
-//        final String offSignatureFile = path + "/training/tplink-bulb/signatures/tplink-bulb-offSignature-phone-side.sig";
-        // TODO: EXPERIMENT - November 16, 2018
-        // TP-Link Bulb experiment
-//        final String inputPcapFile = path + "/experimental_result/standalone/tplink-bulb/wlan1/tplink-bulb.wlan1.local.pcap";
-//        final String inputPcapFile = path + "/experimental_result/standalone/tplink-bulb/eth0/tplink-bulb.eth0.local.pcap";
-//        final String inputPcapFile = path + "/experimental_result/smarthome/tplink-bulb/wlan1/tplink-bulb.wlan1.detection.pcap";
-////        final String inputPcapFile = path + "/experimental_result/smarthome/tplink-bulb/eth0/tplink-bulb.eth0.detection.pcap";
-//        // TP-Link Bulb PHONE signatures
-//        final String onSignatureFile = path + "/experimental_result/standalone/tplink-bulb/signatures/tplink-bulb-onSignature-phone-side.sig";
-//        final String offSignatureFile = path + "/experimental_result/standalone/tplink-bulb/signatures/tplink-bulb-offSignature-phone-side.sig";
-
-        /*
-        // WeMo Plug experiment
-        final String inputPcapFile = path + "/training/wemo-plug/wlan1/wemo-plug.wlan1.local.pcap";
-        // WeMo Plug PHONE signatures
-        final String onSignatureFile = path + "/training/wemo-plug/signatures/wemo-plug-onSignature-device-side.sig";
-        final String offSignatureFile = path + "/training/wemo-plug/signatures/wemo-plug-offSignature-device-side.sig";
-        */
-        // TODO: EXPERIMENT - November 20, 2018
-        // WeMo Plug experiment
-//        final String inputPcapFile = path + "/experimental_result/standalone/wemo-plug/wlan1/wemo-plug.wlan1.local.pcap";
-//        final String inputPcapFile = path + "/experimental_result/standalone/wemo-plug/eth0/wemo-plug.eth0.local.pcap";
-        // TODO: WE HAVE 4 ADDITIONAL EVENTS (TRIGGERED MANUALLY), SO WE JUST IGNORE THEM BECAUSE THEY HAPPENED BEFORE
-        // TODO: THE ACTUAL TRIGGERS
-//        final String inputPcapFile = path + "/experimental_result/smarthome/wemo-plug/wlan1/wemo-plug.wlan1.detection.pcap";
-////        final String inputPcapFile = path + "/experimental_result/smarthome/wemo-plug/eth0/wemo-plug.eth0.detection.pcap";
-//        // WeMo Plug PHONE signatures
-//        final String onSignatureFile = path + "/experimental_result/standalone/wemo-plug/signatures/wemo-plug-onSignature-phone-side.sig";
-//        final String offSignatureFile = path + "/experimental_result/standalone/wemo-plug/signatures/wemo-plug-offSignature-phone-side.sig";
-
-        /*
-        // WeMo Insight Plug experiment
-        final String inputPcapFile = path + "/training/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.local.pcap";
-        // WeMo Insight Plug PHONE signatures
-        final String onSignatureFile = path + "/training/wemo-insight-plug/signatures/wemo-insight-plug-onSignature-device-side.sig";
-        final String offSignatureFile = path + "/training/wemo-insight-plug/signatures/wemo-insight-plug-offSignature-device-side.sig";
-        */
-        // TODO: EXPERIMENT - November 21, 2018
-        // WeMo Insight Plug experiment
-//        final String inputPcapFile = path + "/experimental_result/standalone/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.local.pcap";
-//        final String inputPcapFile = path + "/experimental_result/standalone/wemo-insight-plug/eth0/wemo-insight-plug.eth0.local.pcap";
-        // TODO: WE HAVE 1 ADDITIONAL EVENT (FROM WEMO PLUG)
-//        final String inputPcapFile = path + "/experimental_result/smarthome/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.detection.pcap";
-//        final String inputPcapFile = path + "/experimental_result/smarthome/wemo-insight-plug/eth0/wemo-insight-plug.eth0.detection.pcap";
-        // WeMo Insight Plug PHONE signatures
-//        final String onSignatureFile = path + "/experimental_result/standalone/wemo-insight-plug/signatures/wemo-insight-plug-onSignature-phone-side.sig";
-//        final String offSignatureFile = path + "/experimental_result/standalone/wemo-insight-plug/signatures/wemo-insight-plug-offSignature-phone-side.sig";
-
-
-        // Kwikset Doorlock Sep 12 experiment
-//        final String inputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset3.wlan1.local.pcap";
-//        // Kwikset Doorlock PHONE signatures
-//        final String onSignatureFile = path + "/2018-08/kwikset-doorlock/onSignature-Kwikset-Doorlock-phone.sig";
-//        final String offSignatureFile = path + "/2018-08/kwikset-doorlock/offSignature-Kwikset-Doorlock-phone.sig";
-        // TODO: EXPERIMENT - November 10, 2018
-        // Kwikset Door lock experiment
-//        final String inputPcapFile = path + "/experimental_result/standalone/kwikset-doorlock/wlan1/kwikset-doorlock.wlan1.local.pcap";
-//        //final String inputPcapFile = path + "/experimental_result/smarthome/kwikset-doorlock/wlan1/kwikset-doorlock.wlan1.detection.pcap";
-//        final String inputPcapFile = path + "/experimental_result/smarthome/kwikset-doorlock/eth0/kwikset-doorlock.eth0.detection.pcap";
-////        // Kwikset Door lock PHONE signatures
-//        final String onSignatureFile = path + "/experimental_result/standalone/kwikset-doorlock/signatures/kwikset-doorlock-onSignature-phone-side.sig";
-//        final String offSignatureFile = path + "/experimental_result/standalone/kwikset-doorlock/signatures/kwikset-doorlock-offSignature-phone-side.sig";
-//        final String onSignatureFile = path + "/training/signatures/kwikset-doorlock/kwikset-doorlock-onSignature-phone-side.sig";
-//        final String offSignatureFile = path + "/training/signatures/kwikset-doorlock/kwikset-doorlock-offSignature-phone-side.sig";
-
-
-
-        // D-Link Siren experiment
-//        final String inputPcapFile = path + "/2018-08/dlink-siren/dlink-siren.wlan1.local.pcap";
-        // D-Link Siren DEVICE signatures
-        //final String onSignatureFile = path + "/2018-08/dlink-siren/onSignature-DLink-Siren-device.sig";
-        //final String offSignatureFile = path + "/2018-08/dlink-siren/offSignature-DLink-Siren-device.sig";
-        // D-Link Siren PHONE signatures
-//        final String onSignatureFile = path + "/2018-08/dlink-siren/onSignature-DLink-Siren-phone.sig";
-//        final String offSignatureFile = path + "/2018-08/dlink-siren/offSignature-DLink-Siren-phone.sig";
-
-
-        // Output file names used (to make it easy to catch if one forgets to change them)
-        System.out.println("ON signature file in use is " + onSignatureFile);
-        System.out.println("OFF signature file in use is " + offSignatureFile);
-        System.out.println("PCAP file that is the target of detection is " + inputPcapFile);
-
-        List<List<List<PcapPacket>>> onSignature = PrintUtils.deserializeSignatureFromFile(onSignatureFile);
-        List<List<List<PcapPacket>>> offSignature = PrintUtils.deserializeSignatureFromFile(offSignatureFile);
-
-        // LAN
-//        SignatureDetector onDetector = new SignatureDetector(onSignature, null);
-//        SignatureDetector offDetector = new SignatureDetector(offSignature, null);
-        // WAN
-        SignatureDetector onDetector = new SignatureDetector(onSignature, "128.195.205.105");
-        SignatureDetector offDetector = new SignatureDetector(offSignature, "128.195.205.105");
-
-        final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofLocalizedDateTime(FormatStyle.MEDIUM).
-                withLocale(Locale.US).withZone(ZoneId.of("America/Los_Angeles"));
-
-        // Outputs information about a detected event to std.out
-        final Consumer<UserAction> outputter = ua -> {
-            String eventDescription;
-            switch (ua.getType()) {
-                case TOGGLE_ON:
-                    eventDescription = "ON";
-                    break;
-                case TOGGLE_OFF:
-                    eventDescription = "OFF";
-                    break;
-                default:
-                    throw new AssertionError("unhandled event type");
-            }
-            //String output = String.format("[ !!! %s SIGNATURE DETECTED at %s !!! ]",
-            //      eventDescription, dateTimeFormatter.format(ua.getTimestamp()));
-            String output = String.format("%s",
-                    dateTimeFormatter.format(ua.getTimestamp()));
-            System.out.println(output);
-        };
-
-        // Let's create observers that construct a UserAction representing the detected event.
-        final List<UserAction> detectedEvents = new ArrayList<>();
-        onDetector.addObserver((searched, match) -> {
-            PcapPacket firstPkt = match.get(0).get(0);
-            detectedEvents.add(new UserAction(UserAction.Type.TOGGLE_ON, firstPkt.getTimestamp()));
-        });
-        offDetector.addObserver((searched, match) -> {
-            PcapPacket firstPkt = match.get(0).get(0);
-            detectedEvents.add(new UserAction(UserAction.Type.TOGGLE_OFF, firstPkt.getTimestamp()));
-        });
-
-        PcapHandle handle;
-        try {
-            handle = Pcaps.openOffline(inputPcapFile, PcapHandle.TimestampPrecision.NANO);
-        } catch (PcapNativeException pne) {
-            handle = Pcaps.openOffline(inputPcapFile);
-        }
-        PcapHandleReader reader = new PcapHandleReader(handle, p -> true, onDetector, offDetector);
-        reader.readFromHandle();
-
-        // TODO: need a better way of triggering detection than this...
-        onDetector.mClusterMatchers.forEach(cm -> cm.performDetection());
-        offDetector.mClusterMatchers.forEach(cm -> cm.performDetection());
-
-        // Sort the list of detected events by timestamp to make it easier to compare it line-by-line with the trigger
-        // times file.
-        Collections.sort(detectedEvents, Comparator.comparing(UserAction::getTimestamp));
-
-        // Output the detected events
-        detectedEvents.forEach(outputter);
-
-        System.out.println("Number of detected events of type " + UserAction.Type.TOGGLE_ON + ": " +
-                detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_ON).count());
-        System.out.println("Number of detected events of type " + UserAction.Type.TOGGLE_OFF + ": " +
-                detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_OFF).count());
-
-
-        // TODO: Temporary clean up until we clean the pipeline
-//        List<UserAction> cleanedDetectedEvents = SignatureDetector.removeDuplicates(detectedEvents);
-//        cleanedDetectedEvents.forEach(outputter);
-    }
-
-    /**
-     * The signature that this {@link SignatureDetector} is searching for.
-     */
-    private final List<List<List<PcapPacket>>> mSignature;
-
-    /**
-     * The {@link ClusterMatcher}s in charge of detecting each individual sequence of packets that together make up the
-     * the signature.
-     */
-    private final List<ClusterMatcher> mClusterMatchers;
-
-    /**
-     * For each {@code i} ({@code i >= 0 && i < pendingMatches.length}), {@code pendingMatches[i]} holds the matches
-     * found by the {@link ClusterMatcher} at {@code mClusterMatchers.get(i)} that have yet to be "consumed", i.e.,
-     * have yet to be included in a signature detected by this {@link SignatureDetector} (a signature can be encompassed
-     * of multiple packet sequences occurring shortly after one another on multiple connections).
-     */
-    private final List<List<PcapPacket>>[] pendingMatches;
-
-    /**
-     * Maps a {@link ClusterMatcher} to its corresponding index in {@link #pendingMatches}.
-     */
-    private final Map<ClusterMatcher, Integer> mClusterMatcherIds;
-
-    private final List<SignatureDetectionObserver> mObservers = new ArrayList<>();
-
-    /**
-     * Remove duplicates in {@code List} of {@code UserAction} objects. We need to clean this up for user actions
-     * that appear multiple times.
-     * TODO: This static method is probably just for temporary and we could get rid of this after we clean up
-     * TODO:    the pipeline
-     *
-     * @param listUserAction A {@link List} of {@code UserAction}.
-     *
-     */
-    public static List<UserAction> removeDuplicates(List<UserAction> listUserAction) {
-
-        // Iterate and check for duplicates (check timestamps)
-        Set<Long> epochSecondSet = new HashSet<>();
-        // Create a target list for cleaned up list
-        List<UserAction> listUserActionClean = new ArrayList<>();
-        for(UserAction userAction : listUserAction) {
-            // Don't insert if any duplicate is found
-            if(!epochSecondSet.contains(userAction.getTimestamp().getEpochSecond())) {
-                listUserActionClean.add(userAction);
-                epochSecondSet.add(userAction.getTimestamp().getEpochSecond());
-            }
-        }
-        return listUserActionClean;
-    }
-
-    public SignatureDetector(List<List<List<PcapPacket>>> searchedSignature, String routerWanIp) {
-        // note: doesn't protect inner lists from changes :'(
-        mSignature = Collections.unmodifiableList(searchedSignature);
-        // Generate corresponding/appropriate ClusterMatchers based on the provided signature
-        List<ClusterMatcher> clusterMatchers = new ArrayList<>();
-        for (List<List<PcapPacket>> cluster : mSignature) {
-            clusterMatchers.add(new ClusterMatcher(cluster, routerWanIp, this));
-        }
-        mClusterMatchers = Collections.unmodifiableList(clusterMatchers);
-
-        // < exploratory >
-        pendingMatches = new List[mClusterMatchers.size()];
-        for (int i = 0; i < pendingMatches.length; i++) {
-            pendingMatches[i] = new ArrayList<>();
-        }
-        Map<ClusterMatcher, Integer> clusterMatcherIds = new HashMap<>();
-        for (int i = 0; i < mClusterMatchers.size(); i++) {
-            clusterMatcherIds.put(mClusterMatchers.get(i), i);
-        }
-        mClusterMatcherIds = Collections.unmodifiableMap(clusterMatcherIds);
-    }
-
-    public void addObserver(SignatureDetectionObserver observer) {
-        mObservers.add(observer);
-    }
-
-    public boolean removeObserver(SignatureDetectionObserver observer) {
-        return mObservers.remove(observer);
-    }
-
-    @Override
-    public void gotPacket(PcapPacket packet) {
-        // simply delegate packet reception to all ClusterMatchers.
-        mClusterMatchers.forEach(cm -> cm.gotPacket(packet));
-    }
-
-    @Override
-    public void onMatch(ClusterMatcher clusterMatcher, List<PcapPacket> match) {
-        // Add the match at the corresponding index
-        pendingMatches[mClusterMatcherIds.get(clusterMatcher)].add(match);
-        checkSignatureMatch();
-    }
-
-    private void checkSignatureMatch() {
-        // << Graph-based approach using Balint's idea. >>
-        // This implementation assumes that the packets in the inner lists (the sequences) are ordered by asc timestamp.
-
-        // There cannot be a signature match until each ClusterMatcher has found a match of its respective sequence.
-        if (Arrays.stream(pendingMatches).noneMatch(l -> l.isEmpty())) {
-            // Construct the DAG
-            final SimpleDirectedWeightedGraph<Vertex, DefaultWeightedEdge> graph =
-                    new SimpleDirectedWeightedGraph<>(DefaultWeightedEdge.class);
-            // Add a vertex for each match found by all ClusterMatchers
-            // And maintain an array to keep track of what cluster matcher each vertex corresponds to
-            final List<Vertex>[] vertices = new List[pendingMatches.length];
-            for (int i = 0; i < pendingMatches.length; i++) {
-                vertices[i] = new ArrayList<>();
-                for (List<PcapPacket> sequence : pendingMatches[i]) {
-                    Vertex v = new Vertex(sequence);
-                    vertices[i].add(v); // retain reference for later when we are to add edges
-                    graph.addVertex(v); // add to vertex to graph
-                }
-            }
-            // Add dummy source and sink vertices to facilitate search.
-            final Vertex source = new Vertex(null);
-            final Vertex sink = new Vertex(null);
-            graph.addVertex(source);
-            graph.addVertex(sink);
-            // The source is connected to all vertices that wrap the sequences detected by ClusterMatcher at index 0.
-            // Note: zero cost edges as this is just a dummy link to facilitate search from a common start node.
-            for (Vertex v : vertices[0]) {
-                DefaultWeightedEdge edge = graph.addEdge(source, v);
-                graph.setEdgeWeight(edge, 0.0);
-            }
-            // Similarly, all vertices that wrap the sequences detected by the last ClusterMatcher of the signature
-            // are connected to the sink node.
-            for (Vertex v : vertices[vertices.length-1]) {
-                DefaultWeightedEdge edge = graph.addEdge(v, sink);
-                graph.setEdgeWeight(edge, 0.0);
-            }
-            // Now link sequences detected by ClusterMatcher at index i to sequences detected by ClusterMatcher at index
-            // i+1 if they obey the timestamp constraint (i.e., that the latter is later in time than the former).
-            for (int i = 0; i < vertices.length; i++) {
-                int j = i + 1;
-                if (j < vertices.length) {
-                    for (Vertex iv : vertices[i]) {
-                        PcapPacket ivLast = iv.sequence.get(iv.sequence.size()-1);
-                        for (Vertex jv : vertices[j]) {
-                            PcapPacket jvFirst = jv.sequence.get(jv.sequence.size()-1);
-                            if (ivLast.getTimestamp().isBefore(jvFirst.getTimestamp())) {
-                                DefaultWeightedEdge edge = graph.addEdge(iv, jv);
-                                // The weight is the duration of the i'th sequence plus the duration between the i'th
-                                // and i+1'th sequence.
-                                Duration d = Duration.
-                                        between(iv.sequence.get(0).getTimestamp(), jvFirst.getTimestamp());
-                                // Unfortunately weights are double values, so must convert from long to double.
-                                // TODO: need nano second precision? If so, use d.toNanos().
-                                // TODO: risk of overflow when converting from long to double..?
-                                graph.setEdgeWeight(edge, Long.valueOf(d.toMillis()).doubleValue());
-                            }
-                            // Alternative version if we cannot assume that sequences are ordered by timestamp:
-//                            if (iv.sequence.stream().max(Comparator.comparing(PcapPacket::getTimestamp)).get()
-//                                    .getTimestamp().isBefore(jv.sequence.stream().min(
-//                                            Comparator.comparing(PcapPacket::getTimestamp)).get().getTimestamp())) {
-//
-//                            }
-                        }
-                    }
-                }
-            }
-            // Graph construction complete, run shortest-path to find a (potential) signature match.
-            DijkstraShortestPath<Vertex, DefaultWeightedEdge> dijkstra = new DijkstraShortestPath<>(graph);
-            GraphPath<Vertex, DefaultWeightedEdge> shortestPath = dijkstra.getPath(source, sink);
-            if (shortestPath != null) {
-                // The total weight is the duration between the first packet of the first sequence and the last packet
-                // of the last sequence, so we simply have to compare the weight against the timeframe that we allow
-                // the signature to span. For now we just use the inclusion window we defined for training purposes.
-                // Note however, that we must convert back from double to long as the weight is stored as a double in
-                // JGraphT's API.
-                if (((long)shortestPath.getWeight()) < TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS) {
-                    // There's a signature match!
-                    // Extract the match from the vertices
-                    List<List<PcapPacket>> signatureMatch = new ArrayList<>();
-                    for(Vertex v : shortestPath.getVertexList()) {
-                        if (v == source || v == sink) {
-                            // Skip the dummy source and sink nodes.
-                            continue;
-                        }
-                        signatureMatch.add(v.sequence);
-                        // As there is a one-to-one correspondence between vertices[] and pendingMatches[], we know that
-                        // the sequence we've "consumed" for index i of the matched signature is also at index i in
-                        // pendingMatches. We must remove it from pendingMatches so that we don't use it to construct
-                        // another signature match in a later call.
-                        pendingMatches[signatureMatch.size()-1].remove(v.sequence);
-                    }
-                    // Declare success: notify observers
-                    mObservers.forEach(obs -> obs.onSignatureDetected(mSignature,
-                            Collections.unmodifiableList(signatureMatch)));
-                }
-            }
-        }
-    }
-
-    /**
-     * Used for registering for notifications of signatures detected by a {@link SignatureDetector}.
-     */
-    interface SignatureDetectionObserver {
-
-        /**
-         * Invoked when the {@link SignatureDetector} detects the presence of a signature in the traffic that it's
-         * examining.
-         * @param searchedSignature The signature that the {@link SignatureDetector} reporting the match is searching
-         *                          for.
-         * @param matchingTraffic The actual traffic trace that matches the searched signature.
-         */
-        void onSignatureDetected(List<List<List<PcapPacket>>> searchedSignature,
-                                 List<List<PcapPacket>> matchingTraffic);
-    }
-
-    /**
-     * Encapsulates a {@code List<PcapPacket>} so as to allow the list to be used as a vertex in a graph while avoiding
-     * the expensive {@link AbstractList#equals(Object)} calls when adding vertices to the graph.
-     * Using this wrapper makes the incurred {@code equals(Object)} calls delegate to {@link Object#equals(Object)}
-     * instead of {@link AbstractList#equals(Object)}. The net effect is a faster implementation, but the graph will not
-     * recognize two lists that contain the same items--from a value and not reference point of view--as the same
-     * vertex. However, this is fine for our purposes -- in fact restricting it to reference equality seems more
-     * appropriate.
-     */
-    private static class Vertex {
-        private final List<PcapPacket> sequence;
-        private Vertex(List<PcapPacket> wrappedSequence) {
-            sequence = wrappedSequence;
-        }
-    }
-}
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer2/Layer2ClusterMatcher.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer2/Layer2ClusterMatcher.java
new file mode 100644 (file)
index 0000000..b3a88b0
--- /dev/null
@@ -0,0 +1,150 @@
+package edu.uci.iotproject.detection.layer2;
+
+import edu.uci.iotproject.trafficreassembly.layer2.Layer2FlowReassembler;
+import edu.uci.iotproject.trafficreassembly.layer2.Layer2Flow;
+import edu.uci.iotproject.trafficreassembly.layer2.Layer2FlowReassemblerObserver;
+import edu.uci.iotproject.detection.AbstractClusterMatcher;
+import edu.uci.iotproject.trafficreassembly.layer2.Layer2FlowObserver;
+import edu.uci.iotproject.io.PcapHandleReader;
+import edu.uci.iotproject.util.PrintUtils;
+import org.pcap4j.core.*;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * TODO add class documentation.
+ *
+ * @author Janus Varmarken
+ */
+public class Layer2ClusterMatcher extends AbstractClusterMatcher implements Layer2FlowReassemblerObserver, Layer2FlowObserver {
+
+    public static void main(String[] args) throws PcapNativeException, NotOpenException {
+        final String onSignatureFile = "/Users/varmarken/temp/UCI IoT Project/experiments/training/signatures/tplink-plug/tplink-plug-onSignature-device-side.sig";
+        List<List<List<PcapPacket>>> onSignature = PrintUtils.deserializeSignatureFromFile(onSignatureFile);
+
+
+        Layer2FlowReassembler flowReassembler = new Layer2FlowReassembler();
+
+        Layer2ClusterMatcher l2ClusterMatcher = new Layer2ClusterMatcher(onSignature.get(0));
+        flowReassembler.addObserver(l2ClusterMatcher);
+
+        final String inputPcapFile = "/Users/varmarken/temp/UCI IoT Project/experiments/2018-07/tplink/tplink.wlan1.local.pcap";
+
+        PcapHandle handle;
+        try {
+            handle = Pcaps.openOffline(inputPcapFile, PcapHandle.TimestampPrecision.NANO);
+        } catch (PcapNativeException pne) {
+            handle = Pcaps.openOffline(inputPcapFile);
+        }
+        PcapHandleReader reader = new PcapHandleReader(handle, p -> true, flowReassembler);
+        reader.readFromHandle();
+
+
+    }
+
+
+    private final List<Layer2SequenceMatcher> mSeqMatchers;
+
+    public Layer2ClusterMatcher(List<List<PcapPacket>> cluster) {
+        super(cluster);
+        // Setup a sequence matcher for each sequence of the pruned cluster
+        mSeqMatchers = new ArrayList<>();
+        mCluster.forEach(seq -> mSeqMatchers.add(new Layer2SequenceMatcher(seq)));
+
+//        for (int i = 0; i < mCluster.size(); i++) {
+//
+//
+//            mSeqMatchers[i] = new Layer2SequenceMatcher(mCluster.get(i));
+//
+//
+//        }
+    }
+
+//    @Override
+//    public void gotPacket(PcapPacket packet) {
+//        // Forward the packet to all sequence matchers.
+//        for (Layer2SequenceMatcher matcher : mSeqMatchers) {
+//            matcher.gotPacket(packet);
+//        }
+//
+//
+//    }
+
+
+    private final Map<Layer2Flow, List<Layer2SequenceMatcher>> mPerFlowSeqMatchers = new HashMap<>();
+
+    @Override
+    public void onNewPacket(Layer2Flow flow, PcapPacket newPacket) {
+        if (mPerFlowSeqMatchers.get(flow) == null) {
+            // If this is the first time we encounter this flow, we need to set up sequence matchers for it.
+            List<Layer2SequenceMatcher> matchers = new ArrayList<>();
+            mCluster.forEach(seq -> matchers.add(new Layer2SequenceMatcher(seq)));
+            mPerFlowSeqMatchers.put(flow, matchers);
+        }
+        // Buffer for new sequence matchers that will take over the job of observing for the first packet when a
+        // sequence matcher advances beyond the first packet.
+        List<Layer2SequenceMatcher> newSeqMatchers = new ArrayList<>();
+        // Buffer for sequence matchers that have terminated and are to be removed from mPerFlowSeqMatchers.
+        List<Layer2SequenceMatcher> terminatedSeqMatchers = new ArrayList<>();
+        // Present the new packet to all sequence matchers
+        for (Layer2SequenceMatcher sm : mPerFlowSeqMatchers.get(flow)) {
+            boolean matched = sm.matchPacket(newPacket);
+            if (matched && sm.getMatchedPacketsCount() == 1) {
+                // Setup a new sequence matcher that matches from the beginning of the sequence so as to keep
+                // progressing in the sequence matcher that just matched the current packet, while still allowing
+                // for matches of the full sequence in later traffic. This is to accommodate the case where the
+                // first packet of a sequence is detected in an early packet, but where the remaining packets of
+                // that sequence do not appear until way later in time (e.g., if the first packet of the sequence
+                // by chance is generated from traffic unrelated to the trigger traffic).
+                // Note that we must store the new sequence matcher in a buffer and add it outside the loop in order to
+                // prevent concurrent modification exceptions.
+                newSeqMatchers.add(new Layer2SequenceMatcher(sm.getTargetSequence()));
+            }
+            if (matched && sm.getMatchedPacketsCount() == sm.getTargetSequencePacketCount()) {
+                // This sequence matcher has a match of the sequence it was searching for
+                // TODO report it.... for now just do a dummy printout.
+                System.out.println("SEQUENCE MATCHER HAS A MATCH AT " + sm.getMatchedPackets().get(0).getTimestamp());
+                // Mark the sequence matcher for removal. No need to create a replacement one as we do that whenever the
+                // first packet of the sequence is matched (see above).
+                terminatedSeqMatchers.add(sm);
+            }
+        }
+        // Add the new sequence matchers, if any.
+        mPerFlowSeqMatchers.get(flow).addAll(newSeqMatchers);
+        // Remove the terminated sequence matchers, if any.
+        mPerFlowSeqMatchers.get(flow).removeAll(terminatedSeqMatchers);
+    }
+
+
+    @Override
+    protected List<List<PcapPacket>> pruneCluster(List<List<PcapPacket>> cluster) {
+        // Note: we assume that all sequences in the input cluster are of the same length and that their packet
+        // directions are identical.
+        List<List<PcapPacket>> prunedCluster = new ArrayList<>();
+        for (List<PcapPacket> originalClusterSeq : cluster) {
+            boolean alreadyPresent = prunedCluster.stream().anyMatch(pcPkts -> {
+                for (int i = 0; i < pcPkts.size(); i++) {
+                    if (pcPkts.get(i).getOriginalLength() != originalClusterSeq.get(i).getOriginalLength()) {
+                        return false;
+                    }
+                }
+                return true;
+            });
+            if (!alreadyPresent) {
+                // Add the sequence if not already present in the pruned cluster.
+                prunedCluster.add(originalClusterSeq);
+            }
+        }
+        return prunedCluster;
+    }
+
+
+    @Override
+    public void onNewFlow(Layer2FlowReassembler reassembler, Layer2Flow newFlow) {
+        // Subscribe to the new flow to get updates whenever a new packet pertaining to the flow is processed.
+        newFlow.addFlowObserver(this);
+    }
+}
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SequenceMatcher.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SequenceMatcher.java
new file mode 100644 (file)
index 0000000..10ae34e
--- /dev/null
@@ -0,0 +1,165 @@
+package edu.uci.iotproject.detection.layer2;
+
+import edu.uci.iotproject.analysis.TriggerTrafficExtractor;
+import edu.uci.iotproject.util.PcapPacketUtils;
+import org.pcap4j.core.PcapPacket;
+import org.pcap4j.util.MacAddress;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Attempts to detect the presence of a specific packet sequence in the set of packets provided through multiple calls
+ * to {@link #matchPacket(PcapPacket)}, considering only layer 2 information.
+ *
+ * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
+ * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
+ */
+public class Layer2SequenceMatcher {
+
+    /**
+     * The sequence this {@link Layer2SequenceMatcher} is searching for.
+     */
+    private final List<PcapPacket> mSequence;
+
+    /**
+     * Buffer of actual packets seen so far that match the searched sequence (i.e., constitutes a subsequence of the
+     * searched sequence).
+     */
+    private final List<PcapPacket> mMatchedPackets = new ArrayList<>();
+
+    /**
+     * Models the directions of packets in {@link #mSequence}. As the sequence matcher assumes that it is only presented
+     * with packet from a single flow (packets exchanged between two devices), we can model the packet directions with a
+     * single bit. We don't have any notion "phone to device" or "device to phone" as we don't know the MAC addresses
+     * of devices in advance during matching.
+     */
+    private final boolean[] mPacketDirections;
+
+    /**
+     * Create a {@code Layer2SequenceMatcher}.
+     * @param sequence The sequence to match against (search for).
+     */
+    public Layer2SequenceMatcher(List<PcapPacket> sequence) {
+        mSequence = sequence;
+        // Compute packet directions for sequence.
+        mPacketDirections = new boolean[sequence.size()];
+        for (int i = 0; i < sequence.size(); i++) {
+            if (i == 0) {
+                // No previous packet; boolean parameter is ignored in this special case.
+                mPacketDirections[i] = getPacketDirection(null, true, sequence.get(i));
+            } else {
+                // Base direction marker on direction of previous packet.
+                PcapPacket prevPkt = mSequence.get(i-1);
+                boolean prevPktDirection = mPacketDirections[i-1];
+                mPacketDirections[i] = getPacketDirection(prevPkt, prevPktDirection, sequence.get(i));
+            }
+        }
+    }
+
+    /**
+     * Attempt to advance this {@code Layer2SequenceMatcher} by matching {@code packet} against the packet that this
+     * {@code Layer2SequenceMatcher} expects as the next packet of the sequence it is searching for.
+     * @param packet
+     * @return {@code true} if this {@code Layer2SequenceMatcher} could advance by adding {@code packet} to its set of
+     *         matched packets, {@code false} otherwise.
+     */
+    public boolean matchPacket(PcapPacket packet) {
+        if (getMatchedPacketsCount() == getTargetSequencePacketCount()) {
+            // We already matched the entire sequence, so we can't match any more packets.
+            return false;
+        }
+
+        // Verify that new packet pertains to same flow as previously matched packets, if any.
+        if (getMatchedPacketsCount() > 0) {
+            MacAddress pktSrc = PcapPacketUtils.getEthSrcAddr(packet);
+            MacAddress pktDst = PcapPacketUtils.getEthDstAddr(packet);
+            MacAddress earlierPktSrc = PcapPacketUtils.getEthSrcAddr(mMatchedPackets.get(0));
+            MacAddress earlierPktDst = PcapPacketUtils.getEthDstAddr(mMatchedPackets.get(0));
+            if (!(pktSrc.equals(earlierPktSrc) && pktDst.equals(earlierPktDst) ||
+                    pktSrc.equals(earlierPktDst) && pktDst.equals(earlierPktSrc))) {
+                return false;
+            }
+        }
+
+        // Get representative of the packet we expect to match next.
+        PcapPacket expected = mSequence.get(mMatchedPackets.size());
+        // First verify if the received packet has the length we're looking for.
+        if (packet.getOriginalLength() == expected.getOriginalLength()) {
+            // If this is the first packet, we only need to verify that its length is correct. Time constraints are
+            // obviously satisfied as there are no previous packets. Furthermore, direction matches by definition as we
+            // don't know the MAC of the device (or phone) in advance, so we can't enforce a rule saying "first packet
+            // must originate from this particular MAC".
+            if (getMatchedPacketsCount() == 0) {
+                // Store packet as matched and advance.
+                mMatchedPackets.add(packet);
+                return true;
+            }
+            // Check if direction of packet matches expected direction.
+            boolean actualDirection = getPacketDirection(mMatchedPackets.get(getMatchedPacketsCount()-1),
+                    mPacketDirections[getMatchedPacketsCount()-1], packet);
+            boolean expectedDirection = mPacketDirections[getMatchedPacketsCount()];
+            if (actualDirection != expectedDirection) {
+                return false;
+            }
+            // Next apply timing constraints:
+            // 1: to be a match, the packet must have a later timestamp than any other packet currently matched
+            // 2: does adding the packet cause the max allowed time between first packet and last packet to be exceeded?
+            if (!packet.getTimestamp().isAfter(mMatchedPackets.get(getMatchedPacketsCount()-1).getTimestamp())) {
+                return false;
+            }
+            if (packet.getTimestamp().isAfter(mMatchedPackets.get(0).getTimestamp().
+                            plusMillis(TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS))) {
+                return false;
+            }
+            // If we made it here, it means that this packet has the expected length, direction, and obeys the timing
+            // constraints, so we store it and advance.
+            mMatchedPackets.add(packet);
+            if (mMatchedPackets.size() == mSequence.size()) {
+                // TODO report (to observers?) that we are done?
+            }
+            return true;
+        }
+        return false;
+    }
+
+    public int getMatchedPacketsCount() {
+        return mMatchedPackets.size();
+    }
+
+    public int getTargetSequencePacketCount() {
+        return mSequence.size();
+    }
+
+    public List<PcapPacket> getTargetSequence() {
+        return mSequence;
+    }
+
+    public List<PcapPacket> getMatchedPackets() {
+        return mMatchedPackets;
+    }
+
+    /**
+     * Compute the direction of a packet based on the previous packet. If no previous packet is provided, the direction
+     * of {@code currPkt} is {@code true} by definition.
+     * @param prevPkt The previous packet, if any.
+     * @param prevPktDirection The computed direction of the previous packet
+     * @param currPkt The current packet for which the direction is to be determined.
+     * @return The direction of {@code currPkt}.
+     */
+    private boolean getPacketDirection(PcapPacket prevPkt, boolean prevPktDirection, PcapPacket currPkt) {
+        if (prevPkt == null) {
+            // By definition, use true as direction marker for first packet
+            return true;
+        }
+        if (PcapPacketUtils.getEthSrcAddr(prevPkt).equals(PcapPacketUtils.getEthSrcAddr(currPkt))) {
+            // Current packet goes in same direction as previous packet.
+            return prevPktDirection;
+        } else {
+            // Current packet goes in opposite direction of previous packet.
+            return !prevPktDirection;
+        }
+    }
+
+
+}
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer3/Layer3ClusterMatcher.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer3/Layer3ClusterMatcher.java
new file mode 100644 (file)
index 0000000..f0e3bb6
--- /dev/null
@@ -0,0 +1,353 @@
+package edu.uci.iotproject.detection.layer3;
+
+import edu.uci.iotproject.detection.AbstractClusterMatcher;
+import edu.uci.iotproject.trafficreassembly.layer3.Conversation;
+import edu.uci.iotproject.trafficreassembly.layer3.TcpReassembler;
+import edu.uci.iotproject.analysis.TcpConversationUtils;
+import edu.uci.iotproject.io.PcapHandleReader;
+import edu.uci.iotproject.util.PrintUtils;
+import org.pcap4j.core.*;
+
+import java.time.ZoneId;
+import java.util.*;
+import java.util.stream.Collectors;
+
+import static edu.uci.iotproject.util.PcapPacketUtils.*;
+
+/**
+ * Searches a traffic trace for sequences of packets "belong to" a given cluster (in other words, attempts to classify
+ * traffic as pertaining to a given cluster).
+ *
+ * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
+ * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
+ */
+public class Layer3ClusterMatcher extends AbstractClusterMatcher implements PacketListener {
+
+    // Test client
+    public static void main(String[] args) throws PcapNativeException, NotOpenException {
+
+//        String path = "/scratch/July-2018"; // Rahmadi
+        String path = "/Users/varmarken/temp/UCI IoT Project/experiments"; // Janus
+        final String inputPcapFile = path + "/2018-07/dlink/dlink.wlan1.local.pcap";
+        final String signatureFile = path + "/2018-07/dlink/offSignature1.sig";
+
+        List<List<PcapPacket>> signature = PrintUtils.deserializeClustersFromFile(signatureFile);
+        Layer3ClusterMatcher clusterMatcher = new Layer3ClusterMatcher(signature, null,
+                (sig, match) -> System.out.println(
+                        String.format("[ !!! SIGNATURE DETECTED AT %s !!! ]",
+                                match.get(0).getTimestamp().atZone(ZoneId.of("America/Los_Angeles")))
+                )
+        );
+
+        PcapHandle handle;
+        try {
+            handle = Pcaps.openOffline(inputPcapFile, PcapHandle.TimestampPrecision.NANO);
+        } catch (PcapNativeException pne) {
+            handle = Pcaps.openOffline(inputPcapFile);
+        }
+        PcapHandleReader reader = new PcapHandleReader(handle, p -> true, clusterMatcher);
+        reader.readFromHandle();
+        clusterMatcher.performDetection();
+    }
+
+    /**
+     * The ordered directions of packets in the sequences that make up {@link #mCluster}.
+     */
+    private final Conversation.Direction[] mClusterMemberDirections;
+
+    /**
+     * For reassembling the observed traffic into TCP connections.
+     */
+    private final TcpReassembler mTcpReassembler = new TcpReassembler();
+
+    /**
+     * IP of the router's WAN port (if analyzed traffic is captured at the ISP's point of view).
+     */
+    private final String mRouterWanIp;
+
+    private final ClusterMatchObserver[] mObservers;
+
+    /**
+     * Create a {@link Layer3ClusterMatcher}.
+     * @param cluster The cluster that traffic is matched against.
+     * @param routerWanIp The router's WAN IP if examining traffic captured at the ISP's point of view (used for
+     *                    determining the direction of packets).
+     * @param detectionObservers Client code that wants to get notified whenever the {@link Layer3ClusterMatcher} detects that
+     *                          (a subset of) the examined traffic is similar to the traffic that makes up
+     *                          {@code cluster}, i.e., when the examined traffic is classified as pertaining to
+     *                          {@code cluster}.
+     */
+    public Layer3ClusterMatcher(List<List<PcapPacket>> cluster, String routerWanIp, ClusterMatchObserver... detectionObservers) {
+        super(cluster);
+        // ===================== PRECONDITION SECTION =====================
+        mObservers = Objects.requireNonNull(detectionObservers, "detectionObservers cannot be null");
+        if (mObservers.length == 0) {
+            throw new IllegalArgumentException("no detectionObservers provided");
+        }
+        // Build the cluster members' direction sequence.
+        // Note: assumes that the provided cluster was captured within the local network (routerWanIp is set to null).
+        mClusterMemberDirections = getPacketDirections(cluster.get(0), null);
+        /*
+         * Enforce restriction on cluster members: all representatives must exhibit the same direction pattern and
+         * contain the same number of packets. Note that this is a somewhat heavy operation, so it may be disabled later
+         * on in favor of performance. However, it is only run once (at instantiation), so the overhead may be warranted
+         * in order to ensure correctness, especially during the development/debugging phase.
+         */
+        if (mCluster.stream().
+                anyMatch(inner -> !Arrays.equals(mClusterMemberDirections, getPacketDirections(inner, null)))) {
+            throw new IllegalArgumentException(
+                    "cluster members must contain the same number of packets and exhibit the same packet direction " +
+                            "pattern"
+            );
+        }
+        // ================================================================
+        mRouterWanIp = routerWanIp;
+    }
+
+    @Override
+    public void gotPacket(PcapPacket packet) {
+        // Present packet to TCP reassembler so that it can be mapped to a connection (if it is a TCP packet).
+        mTcpReassembler.gotPacket(packet);
+    }
+
+    /**
+     * Get the cluster that describes the packet sequence that this {@link Layer3ClusterMatcher} is searching for.
+     * @return the cluster that describes the packet sequence that this {@link Layer3ClusterMatcher} is searching for.
+     */
+    public List<List<PcapPacket>> getCluster() {
+        return mCluster;
+    }
+
+    public void performDetection() {
+        /*
+         * Let's start out simple by building a version that only works for signatures that do not span across multiple
+         * TCP conversations...
+         */
+        for (Conversation c : mTcpReassembler.getTcpConversations()) {
+            if (c.isTls() && c.getTlsApplicationDataPackets().isEmpty() || !c.isTls() && c.getPackets().isEmpty()) {
+                // Skip empty conversations.
+                continue;
+            }
+            for (List<PcapPacket> signatureSequence : mCluster) {
+                if (isTlsSequence(signatureSequence) != c.isTls()) {
+                    // We consider it a mismatch if one is a TLS application data sequence and the other is not.
+                    continue;
+                }
+                // Fetch set of packets to examine based on TLS or not.
+                List<PcapPacket> cPkts = c.isTls() ? c.getTlsApplicationDataPackets() : c.getPackets();
+                /*
+                 * Note: we embed the attempt to detect the signature sequence in a loop in order to capture those cases
+                 * where the same signature sequence appears multiple times in one Conversation.
+                 *
+                 * Note: since we expect all sequences that together make up the signature to exhibit the same direction
+                 * pattern, we can simply pass the precomputed direction array for the signature sequence so that it
+                 * won't have to be recomputed internally in each call to findSubsequenceInSequence().
+                 */
+                Optional<List<PcapPacket>> match;
+                while ((match = findSubsequenceInSequence(signatureSequence, cPkts, mClusterMemberDirections, null)).
+                        isPresent()) {
+                    List<PcapPacket> matchSeq = match.get();
+                    // Notify observers about the match.
+                    Arrays.stream(mObservers).forEach(o -> o.onMatch(Layer3ClusterMatcher.this, matchSeq));
+                    /*
+                     * Get the index in cPkts of the last packet in the sequence of packets that matches the searched
+                     * signature sequence.
+                     */
+                    int matchSeqEndIdx = cPkts.indexOf(matchSeq.get(matchSeq.size()-1));
+                    // We restart the search for the signature sequence immediately after that index, so truncate cPkts.
+                    cPkts = cPkts.stream().skip(matchSeqEndIdx + 1).collect(Collectors.toList());
+                }
+            }
+            /*
+             * TODO:
+             * if no item in cluster matches, also perform a distance-based matching to cover those cases where we did
+             * not manage to capture every single mutation of the sequence during training.
+             *
+             * Need to compute average/centroid of cluster to do so...? Compute within-cluster variance, then check if
+             * distance between input conversation and cluster average/centroid is smaller than or equal to the computed
+             * variance?
+             */
+        }
+    }
+
+    /**
+     * Checks if {@code sequence} is a sequence of TLS packets. Note: the current implementation relies on inspection
+     * of the port numbers when deciding between TLS vs. non-TLS. Therefore, only the first packet of {@code sequence}
+     * is examined as it is assumed that all packets in {@code sequence} pertain to the same {@link Conversation} and
+     * hence share the same set of two src/dst port numbers (albeit possibly alternating between which one is the src
+     * and which one is the dst, as packets in {@code sequence} may be in alternating directions).
+     * @param sequence The sequence of packets for which it is to be determined if it is a sequence of TLS packets or
+     *                 non-TLS packets.
+     * @return {@code true} if {@code sequence} is a sequence of TLS packets, {@code false} otherwise.
+     */
+    private boolean isTlsSequence(List<PcapPacket> sequence) {
+        // NOTE: Assumes ALL packets in sequence pertain to the same TCP connection!
+        PcapPacket firstPkt = sequence.get(0);
+        int srcPort = getSourcePort(firstPkt);
+        int dstPort = getDestinationPort(firstPkt);
+        return TcpConversationUtils.isTlsPort(srcPort) || TcpConversationUtils.isTlsPort(dstPort);
+    }
+
+    /**
+     * Examine if a given sequence of packets ({@code sequence}) contains a given shorter sequence of packets
+     * ({@code subsequence}). Note: the current implementation actually searches for a substring as it does not allow
+     * for interleaving packets in {@code sequence} that are not in {@code subsequence}; for example, if
+     * {@code subsequence} consists of packet lengths [2, 3, 5] and {@code sequence} consists of  packet lengths
+     * [2, 3, 4, 5], the result will be that there is no match (because of the interleaving 4). If we are to allow
+     * interleaving packets, we need a modified version of
+     * <a href="https://stackoverflow.com/a/20545604/1214974">this</a>.
+     *
+     * @param subsequence The sequence to search for.
+     * @param sequence The sequence to search.
+     * @param subsequenceDirections The directions of packets in {@code subsequence} such that for all {@code i},
+     *                              {@code subsequenceDirections[i]} is the direction of the packet returned by
+     *                              {@code subsequence.get(i)}. May be set to {@code null}, in which this call will
+     *                              internally compute the packet directions.
+     * @param sequenceDirections The directions of packets in {@code sequence} such that for all {@code i},
+     *                           {@code sequenceDirections[i]} is the direction of the packet returned by
+     *                           {@code sequence.get(i)}. May be set to {@code null}, in which this call will internally
+     *                           compute the packet directions.
+     *
+     * @return An {@link Optional} containing the part of {@code sequence} that matches {@code subsequence}, or an empty
+     *         {@link Optional} if no part of {@code sequence} matches {@code subsequence}.
+     */
+    private Optional<List<PcapPacket>> findSubsequenceInSequence(List<PcapPacket> subsequence,
+                                                                 List<PcapPacket> sequence,
+                                                                 Conversation.Direction[] subsequenceDirections,
+                                                                 Conversation.Direction[] sequenceDirections) {
+        if (sequence.size() < subsequence.size()) {
+            // If subsequence is longer, it cannot be contained in sequence.
+            return Optional.empty();
+        }
+        if (isTlsSequence(subsequence) != isTlsSequence(sequence)) {
+            // We consider it a mismatch if one is a TLS application data sequence and the other is not.
+            return Optional.empty();
+        }
+        // If packet directions have not been precomputed by calling code, we need to construct them.
+        if (subsequenceDirections == null) {
+            subsequenceDirections = getPacketDirections(subsequence, mRouterWanIp);
+        }
+        if (sequenceDirections == null) {
+            sequenceDirections = getPacketDirections(sequence, mRouterWanIp);
+        }
+        int subseqIdx = 0;
+        int seqIdx = 0;
+        while (seqIdx < sequence.size()) {
+            PcapPacket subseqPkt = subsequence.get(subseqIdx);
+            PcapPacket seqPkt = sequence.get(seqIdx);
+            // We only have a match if packet lengths and directions match.
+            if (subseqPkt.getOriginalLength() == seqPkt.getOriginalLength() &&
+                    subsequenceDirections[subseqIdx] == sequenceDirections[seqIdx]) {
+                // A match; advance both indices to consider next packet in subsequence vs. next packet in sequence.
+                subseqIdx++;
+                seqIdx++;
+                if (subseqIdx == subsequence.size()) {
+                    // We managed to match the entire subsequence in sequence.
+                    // Return the sublist of sequence that matches subsequence.
+                    /*
+                     * TODO:
+                     * ASSUMES THE BACKING LIST (i.e., 'sequence') IS _NOT_ STRUCTURALLY MODIFIED, hence may not work
+                     * for live traces!
+                     */
+                    return Optional.of(sequence.subList(seqIdx - subsequence.size(), seqIdx));
+                }
+            } else {
+                // Mismatch.
+                if (subseqIdx > 0) {
+                    /*
+                     * If we managed to match parts of subsequence, we restart the search for subsequence in sequence at
+                     * the index of sequence where the current mismatch occurred. I.e., we must reset subseqIdx, but
+                     * leave seqIdx untouched.
+                     */
+                    subseqIdx = 0;
+                } else {
+                    /*
+                     * First packet of subsequence didn't match packet at seqIdx of sequence, so we move forward in
+                     * sequence, i.e., we continue the search for subsequence in sequence starting at index seqIdx+1 of
+                     * sequence.
+                     */
+                    seqIdx++;
+                }
+            }
+        }
+        return Optional.empty();
+    }
+
+    /**
+     * Given a cluster, produces a pruned version of that cluster. In the pruned version, there are no duplicate cluster
+     * members. Two cluster members are considered identical if their packets lengths and packet directions are
+     * identical. The resulting pruned cluster is unmodifiable (this applies to both the outermost list as well as the
+     * nested lists) in order to preserve its integrity when exposed to external code (e.g., through
+     * {@link #getCluster()}).
+     *
+     * @param cluster A cluster to prune.
+     * @return The resulting pruned cluster.
+     */
+    @Override
+    protected List<List<PcapPacket>> pruneCluster(List<List<PcapPacket>> cluster) {
+        List<List<PcapPacket>> prunedCluster = new ArrayList<>();
+        for (List<PcapPacket> originalClusterSeq : cluster) {
+            boolean alreadyPresent = false;
+            for (List<PcapPacket> prunedClusterSeq : prunedCluster) {
+                Optional<List<PcapPacket>> duplicate = findSubsequenceInSequence(originalClusterSeq, prunedClusterSeq,
+                        mClusterMemberDirections, mClusterMemberDirections);
+                if (duplicate.isPresent()) {
+                    alreadyPresent = true;
+                    break;
+                }
+            }
+            if (!alreadyPresent) {
+                prunedCluster.add(Collections.unmodifiableList(originalClusterSeq));
+            }
+        }
+        return Collections.unmodifiableList(prunedCluster);
+    }
+
+    /**
+     * Given a {@code List<PcapPacket>}, generate a {@code Conversation.Direction[]} such that each entry in the
+     * resulting {@code Conversation.Direction[]} specifies the direction of the {@link PcapPacket} at the corresponding
+     * index in the input list.
+     * @param packets The list of packets for which to construct a corresponding array of packet directions.
+     * @param routerWanIp The IP of the router's WAN port. This is used for determining the direction of packets when
+     *                    the traffic is captured just outside the local network (at the ISP side of the router). Set to
+     *                    {@code null} if {@code packets} stem from traffic captured within the local network.
+     * @return A {@code Conversation.Direction[]} specifying the direction of the {@link PcapPacket} at the
+     *         corresponding index in {@code packets}.
+     */
+    private static Conversation.Direction[] getPacketDirections(List<PcapPacket> packets, String routerWanIp) {
+        Conversation.Direction[] directions = new Conversation.Direction[packets.size()];
+        for (int i = 0; i < packets.size(); i++) {
+            PcapPacket pkt = packets.get(i);
+            if (getSourceIp(pkt).equals(getDestinationIp(pkt))) {
+                // Sanity check: we shouldn't be processing loopback traffic
+                throw new AssertionError("loopback traffic detected");
+            }
+            if (isSrcIpLocal(pkt) || getSourceIp(pkt).equals(routerWanIp)) {
+                directions[i] = Conversation.Direction.CLIENT_TO_SERVER;
+            } else if (isDstIpLocal(pkt) || getDestinationIp(pkt).equals(routerWanIp)) {
+                directions[i] = Conversation.Direction.SERVER_TO_CLIENT;
+            } else {
+                //throw new IllegalArgumentException("no local IP or router WAN port IP found, can't detect direction");
+            }
+        }
+        return directions;
+    }
+
+    /**
+     * Interface used by client code to register for receiving a notification whenever the {@link Layer3ClusterMatcher}
+     * detects traffic that is similar to the traffic that makes up the cluster returned by
+     * {@link Layer3ClusterMatcher#getCluster()}.
+     */
+    interface ClusterMatchObserver {
+        /**
+         * Callback that is invoked whenever a sequence that is similar to a sequence associated with the cluster (i.e.,
+         * a sequence is a member of the cluster) is detected in the traffic that the associated {@link Layer3ClusterMatcher}
+         * observes.
+         * @param clusterMatcher The {@link Layer3ClusterMatcher} that detected a match (classified traffic as pertaining to
+         *                       its associated cluster).
+         * @param match The traffic that was deemed to match the cluster associated with {@code clusterMatcher}.
+         */
+        void onMatch(Layer3ClusterMatcher clusterMatcher, List<PcapPacket> match);
+    }
+
+}
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer3/SignatureDetector.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer3/SignatureDetector.java
new file mode 100644 (file)
index 0000000..c1a5a9c
--- /dev/null
@@ -0,0 +1,660 @@
+package edu.uci.iotproject.detection.layer3;
+
+import edu.uci.iotproject.analysis.TriggerTrafficExtractor;
+import edu.uci.iotproject.analysis.UserAction;
+import edu.uci.iotproject.io.PcapHandleReader;
+import edu.uci.iotproject.util.PrintUtils;
+import org.jgrapht.GraphPath;
+import org.jgrapht.alg.shortestpath.DijkstraShortestPath;
+import org.jgrapht.graph.DefaultWeightedEdge;
+import org.jgrapht.graph.SimpleDirectedWeightedGraph;
+import org.pcap4j.core.*;
+
+import java.time.Duration;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.time.format.FormatStyle;
+import java.util.*;
+import java.util.function.Consumer;
+
+/**
+ * Detects an event signature that spans one or multiple TCP connections.
+ *
+ * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
+ * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
+ */
+public class SignatureDetector implements PacketListener, Layer3ClusterMatcher.ClusterMatchObserver {
+
+    // Test client
+    public static void main(String[] args) throws PcapNativeException, NotOpenException {
+//        if (args.length < 3) {
+//            String errMsg = String.format("Usage: %s inputPcapFile onSignatureFile offSignatureFile",
+//                    SignatureDetector.class.getSimpleName());
+//            System.out.println(errMsg);
+//            return;
+//        }
+//        final String inputPcapFile = args[0];
+//        final String onSignatureFile = args[1];
+//        final String offSignatureFile = args[2];
+
+        String path = "/scratch/July-2018"; // Rahmadi
+//        String path = "/Users/varmarken/temp/UCI IoT Project/experiments"; // Janus
+//        String path = "/home/jvarmark/iot_project/datasets"; // Hera (server)
+//        String path = "/raid/varmarken/iot_project/datasets"; // Zeus (server)
+
+        // No activity test
+        //final String inputPcapFile = path + "/evaluation/no-activity/no-activity.wlan1.pcap";
+
+        // D-Link Siren experiment
+//        final String inputPcapFile = path + "/evaluation/dlink-siren/dlink-siren.data.wlan1.pcap";
+//        final String inputPcapFile = path + "/evaluation/dlink-siren/dlink-siren.eth0.local.pcap";
+        // D-Link Siren DEVICE signatures
+//        final String onSignatureFile = path + "/2018-08/dlink-siren/onSignature-DLink-Siren-device.sig";
+//        final String offSignatureFile = path + "/2018-08/dlink-siren/offSignature-DLink-Siren-device.sig";
+        // D-Link Siren PHONE signatures
+//        final String onSignatureFile = path + "/2018-08/dlink-siren/onSignature-DLink-Siren-phone.sig";
+//        final String offSignatureFile = path + "/2018-08/dlink-siren/offSignature-DLink-Siren-phone.sig";
+        // TODO: EXPERIMENT - November 19, 2018
+        // Hue Bulb experiment
+//        final String inputPcapFile = path + "/2018-08/hue-bulb/hue-bulb.wlan1.local.pcap";
+        // Hue Bulb PHONE signatures
+//        final String onSignatureFile = path + "/experimental_result/standalone/hue-bulb/signatures/hue-bulb-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/hue-bulb/signatures/hue-bulb-offSignature-phone-side.sig";
+
+        /*
+        // Kwikset Doorlock Sep 12 experiment
+//        final String inputPcapFile = path + "/evaluation/kwikset-doorlock/kwikset-doorlock.data.wlan1.pcap";
+        final String inputPcapFile = path + "/evaluation/kwikset-doorlock/kwikset-doorlock.data.eth0.pcap";
+//        // Kwikset Doorlock PHONE signatures
+        final String onSignatureFile = path + "/2018-08/kwikset-doorlock/onSignature-Kwikset-Doorlock-phone-new.sig";
+        final String offSignatureFile = path + "/2018-08/kwikset-doorlock/offSignature-Kwikset-Doorlock-phone-new.sig";
+        */
+
+        // D-Link Plug experiment
+        //final String inputPcapFile = path + "/evaluation/dlink/dlink-plug.data.wlan1.pcap";
+//        final String inputPcapFile = path + "/evaluation/dlink/dlink-plug.data.eth0.pcap";
+
+        // D-Link Plug DEVICE signatures
+//        final String onSignatureFile = path + "/2018-07/dlink/onSignature-DLink-Plug-device.sig";
+//        final String offSignatureFile = path + "/2018-07/dlink/offSignature-DLink-Plug-device.sig";
+        // D-Link Plug PHONE signatures
+//        final String onSignatureFile = path + "/2018-07/dlink/onSignature-DLink-Plug-phone.sig";
+//        final String offSignatureFile = path + "/2018-07/dlink/offSignature-DLink-Plug-phone.sig";
+
+        // TODO: The following are negative tests against the PCAP file from UNSW
+//        final String inputPcapFile = path + "/UNSW/16-10-04.pcap"; // TODO: Seems to be broken! Zero-payload!
+//        final String inputPcapFile = path + "/UNSW/16-10-12.pcap";
+
+//        final String inputPcapFile = path + "/UNSW/16-09-28.pcap"; // TODO: Seems to be broken! Zero-payload!
+//        final String inputPcapFile = path + "/UNSW/16-10-02.pcap"; // TODO: Seems to be broken!
+//        final String inputPcapFile = path + "/UNSW/16-10-03.pcap"; // TODO: Seems to be broken!
+//        final String inputPcapFile = path + "/UNSW/16-10-04-a.pcap"; // TODO: Seems to be broken! Zero-payload!
+//        final String inputPcapFile = path + "/UNSW/16-10-04-b.pcap"; // TODO: Seems to be broken! Zero-payload!
+//        final String inputPcapFile = path + "/UNSW/16-10-07.pcap"; // TODO: Seems to be broken!
+//        final String inputPcapFile = path + "/UNSW/16-10-08.pcap"; // TODO: Seems to be broken!
+//        final String inputPcapFile = path + "/UNSW/16-10-09.pcap"; // TODO: Seems to be broken!
+//        final String inputPcapFile = path + "/UNSW/16-10-10.pcap"; // TODO: Seems to be broken!
+//        final String inputPcapFile = path + "/UNSW/16-10-11.pcap"; // TODO: Seems to be broken!
+        // TODO: The following one is very long!!! - Split into smaller files!
+//        final String inputPcapFile = path + "/UNSW/16-10-12-a.pcap";
+//        final String inputPcapFile = path + "/UNSW/16-10-12-b.pcap";
+//        final String inputPcapFile = path + "/UNSW/16-10-12-c.pcap";
+//        final String inputPcapFile = path + "/UNSW/16-10-12-d.pcap";
+
+//        final String inputPcapFile = path + "/UNSW/16-09-23.pcap";
+//        final String inputPcapFile = path + "/UNSW/16-09-24.pcap";
+//        final String inputPcapFile = path + "/UNSW/16-09-25.pcap";
+//        final String inputPcapFile = path + "/UNSW/16-09-26.pcap";
+//        final String inputPcapFile = path + "/UNSW/16-09-27.pcap";
+//        final String inputPcapFile = path + "/UNSW/16-09-29.pcap";
+//        final String inputPcapFile = path + "/UNSW/16-10-01.pcap";
+//        final String inputPcapFile = path + "/UNSW/16-10-06.pcap";
+        // Negative test: dataset from UNB
+//        final String inputPcapFile = path + "/evaluation/negative-datasets/UNB/Monday-WorkingHours_one-local-endpoint.pcap";
+
+        // TODO: The following are tests for signatures against training data
+
+        // D-Link Plug experiment
+//        final String inputPcapFile = path + "/training/dlink-plug/wlan1/dlink-plug.wlan1.local.pcap";
+        // D-Link Plug DEVICE signatures
+//        final String onSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-onSignature-device-side.sig";
+//        final String offSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-offSignature-device-side.sig";
+        // D-Link Plug PHONE signatures
+//        final String onSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-offSignature-phone-side.sig";
+
+        // TODO: EXPERIMENT - November 7, 2018
+        // D-Link Plug experiment
+        //final String inputPcapFile = path + "/experimental_result/standalone/dlink-plug/wlan1/dlink-plug.wlan1.local.pcap";
+        //final String inputPcapFile = path + "/experimental_result/smarthome/dlink-plug/wlan1/dlink-plug.wlan1.detection.pcap";
+        //final String inputPcapFile = path + "/experimental_result/smarthome/dlink-plug/eth0/dlink-plug.eth0.detection.pcap";
+        // D-Link Plug DEVICE signatures
+//        final String onSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-onSignature-device-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-offSignature-device-side.sig";
+        // D-Link Plug PHONE signatures
+//        final String onSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-offSignature-phone-side.sig";
+
+        // TODO: EXPERIMENT - November 9, 2018
+        // D-Link Siren experiment
+        //final String inputPcapFile = path + "/experimental_result/standalone/dlink-siren/wlan1/dlink-siren.wlan1.local.pcap";
+        //final String inputPcapFile = path + "/experimental_result/smarthome/dlink-siren/wlan1/dlink-siren.wlan1.detection.pcap";
+//        final String inputPcapFile = path + "/experimental_result/smarthome/dlink-siren/eth0/dlink-siren.eth0.detection.pcap";
+        // D-Link Siren DEVICE signatures
+        // TODO: The device signature does not have pairs---only one packet which is 216, so we don't consider this as a signature
+//        final String onSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-onSignature-device-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-offSignature-device-side.sig";
+        // D-Link Siren PHONE signatures
+//        final String onSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-offSignature-phone-side.sig";
+//        final String onSignatureFile = path + "/training/signatures/dlink-siren/dlink-siren-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/training/signatures/dlink-siren/dlink-siren-offSignature-phone-side.sig";
+
+        // TP-Link Plug experiment
+////        final String inputPcapFile = path + "/training/tplink-plug/wlan1/tplink-plug.wlan1.local.pcap";
+////        final String inputPcapFile = path + "/experimental_result/wifi-Sniffer/tests2/airtool_2019-01-04_11.08.45.AM.pcap";
+//        final String inputPcapFile = path + "/experimental_result/wifi-Sniffer/tests2/command-frames-only.pcap";
+//        // TP-Link Plug DEVICE signatures
+//        final String onSignatureFile = path + "/training/tplink-plug/signatures/tplink-plug-onSignature-device-side.sig";
+//        final String offSignatureFile = path + "/training/tplink-plug/signatures/tplink-plug-offSignature-device-side.sig";
+        // TODO: EXPERIMENT - November 8, 2018
+        // TP-Link Plug experiment
+//        final String inputPcapFile = path + "/experimental_result/standalone/tplink-plug/wlan1/tplink-plug.wlan1.local.pcap";
+//        final String inputPcapFile = path + "/experimental_result/standalone/tplink-plug/eth0/tplink-plug.eth0.local.pcap";
+//        final String inputPcapFile = path + "/experimental_result/smarthome/tplink-plug/wlan1/tplink-plug.wlan1.detection.pcap";
+        //final String inputPcapFile = path + "/experimental_result/smarthome/tplink-plug/eth0/tplink-plug.eth0.detection.pcap";
+        // TP-Link Plug DEVICE signatures
+//        final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-device-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-device-side.sig";
+//        final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-device-side-outbound.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-device-side-outbound.sig";
+        // TP-Link Plug PHONE signatures
+//        final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-phone-side.sig";
+
+        // Arlo camera experiment
+//        final String inputPcapFile = path + "/training/arlo-camera/wlan1/arlo-camera.wlan1.local.pcap";
+////        // TP-Link Plug DEVICE signatures
+//        final String onSignatureFile = path + "/training/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/training/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig";
+        // TODO: EXPERIMENT - November 13, 2018
+        // Arlo Camera experiment
+//        final String inputPcapFile = path + "/experimental_result/standalone/arlo-camera/wlan1/arlo-camera.wlan1.local.pcap";
+//        final String inputPcapFile = path + "/experimental_result/standalone/arlo-camera/eth0/arlo-camera.eth0.local.pcap";
+//        final String inputPcapFile = path + "/experimental_result/smarthome/arlo-camera/wlan1/arlo-camera.wlan1.detection.pcap";
+//        final String inputPcapFile = path + "/experimental_result/smarthome/arlo-camera/eth0/arlo-camera.eth0.detection.pcap";
+//        final String inputPcapFile = path + "/training/arlo-camera/eth0/arlo-camera.eth0.local.pcap";
+        // Arlo Camera PHONE signatures
+//        final String onSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig";
+
+        // Amazon Alexa experiment
+//        final String inputPcapFile = path + "/training/amazon-alexa/wlan1/alexa2.wlan1.local.pcap";
+//        // TP-Link Plug DEVICE signatures
+//        final String onSignatureFile = path + "/training/amazon-alexa/signatures/amazon-alexa-onSignature-device-side.sig";
+//        final String offSignatureFile = path + "/training/amazon-alexa/signatures/amazon-alexa-offSignature-device-side.sig";
+
+        // SmartThings Plug experiment
+//        final String inputPcapFile = path + "/training/st-plug/wlan1/st-plug.wlan1.local.pcap";
+//        // SmartThings Plug DEVICE signatures
+//        //final String onSignatureFile = path + "/training/st-plug/signatures/st-plug-onSignature-device-side.sig";
+//        //final String offSignatureFile = path + "/training/st-plug/signatures/st-plug-offSignature-device-side.sig";
+//        // SmartThings Plug PHONE signatures
+//        final String onSignatureFile = path + "/training/st-plug/signatures/st-plug-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/training/st-plug/signatures/st-plug-offSignature-phone-side.sig";
+        // TODO: EXPERIMENT - November 12, 2018
+        // SmartThings Plug experiment
+//        final String inputPcapFile = path + "/experimental_result/standalone/st-plug/wlan1/st-plug.wlan1.local.pcap";
+//        final String inputPcapFile = path + "/experimental_result/standalone/st-plug/eth0/st-plug.eth0.local.pcap";
+//        //final String inputPcapFile = path + "/experimental_result/smarthome/st-plug/wlan1/st-plug.wlan1.detection.pcap";
+//        final String inputPcapFile = path + "/experimental_result/smarthome/st-plug/eth0/st-plug.eth0.detection.pcap";
+//        // SmartThings Plug PHONE signatures
+//        final String onSignatureFile = path + "/experimental_result/standalone/st-plug/signatures/st-plug-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/st-plug/signatures/st-plug-offSignature-phone-side.sig";
+//        final String onSignatureFile = path + "/training/signatures/st-plug/st-plug-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/training/signatures/st-plug/st-plug-offSignature-phone-side.sig";
+
+        // TODO: EXPERIMENT - January 9, 2018
+        // Blossom Sprinkler experiment
+//        final String inputPcapFile = path + "/experimental_result/standalone/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.local.pcap";
+        final String inputPcapFile = path + "/experimental_result/smarthome/blossom-sprinkler/eth0/blossom-sprinkler.eth0.detection.pcap";
+//        final String inputPcapFile = path + "/experimental_result/smarthome/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.detection.pcap";
+        // Blossom Sprinkler DEVICE signatures
+        final String onSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig";
+        final String offSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig";
+        // Blossom Sprinkler PHONE signatures
+//        final String onSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-phone-side.sig";
+
+        // LiFX Bulb experiment
+//        final String inputPcapFile = path + "/training/lifx-bulb/wlan1/lifx-bulb.wlan1.local.pcap";
+//        // LiFX Bulb DEVICE signatures
+//        final String onSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-onSignature-device-side.sig";
+//        final String offSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-offSignature-device-side.sig";
+        // LiFX Bulb PHONE signatures
+//        final String onSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-offSignature-phone-side.sig";
+
+        // Blossom Sprinkler experiment
+//        //final String inputPcapFile = path + "/training/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.local.pcap";
+//        final String inputPcapFile = path + "/2018-08/blossom/blossom.wlan1.local.pcap";
+//        //final String inputPcapFile = path + "/training/blossom-sprinkler/eth0/blossom-sprinkler.eth0.local.pcap";
+//        // Blossom Sprinkler DEVICE signatures
+//        final String onSignatureFile = path + "/training/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig";
+//        final String offSignatureFile = path + "/training/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig";
+
+        // Nest Thermostat experiment
+//        final String inputPcapFile = path + "/training/nest-thermostat/wlan1/nest-thermostat.wlan1.local.pcap";
+//        // Nest Thermostat DEVICE signatures
+////        final String onSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-onSignature-device-side.sig";
+////        final String offSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-offSignature-device-side.sig";
+//        // Nest Thermostat PHONE signatures
+//        final String onSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-offSignature-phone-side.sig";
+        // TODO: EXPERIMENT - November 15, 2018
+        // Nest Thermostat experiment
+//        final String inputPcapFile = path + "/experimental_result/standalone/nest-thermostat/wlan1/nest-thermostat.wlan1.local.pcap";
+////        final String inputPcapFile = path + "/experimental_result/standalone/nest-thermostat/eth0/nest-thermostat.eth0.local.pcap";
+////        final String inputPcapFile = path + "/experimental_result/smarthome/nest-thermostat/wlan1/nest-thermostat.wlan1.detection.pcap";
+//        final String inputPcapFile = path + "/experimental_result/smarthome/nest-thermostat/eth0/nest-thermostat.eth0.detection.pcap";
+////        // Nest Thermostat PHONE signatures
+//        final String onSignatureFile = path + "/experimental_result/standalone/nest-thermostat/signatures/nest-thermostat-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/nest-thermostat/signatures/nest-thermostat-offSignature-phone-side.sig";
+
+        /*
+        // Hue Bulb experiment
+        final String inputPcapFile = path + "/training/hue-bulb/wlan1/hue-bulb.wlan1.local.pcap";
+        // Hue Bulb PHONE signatures
+        final String onSignatureFile = path + "/training/hue-bulb/signatures/hue-bulb-onSignature-phone-side.sig";
+        final String offSignatureFile = path + "/training/hue-bulb/signatures/hue-bulb-offSignature-phone-side.sig";
+        */
+
+
+
+        // TP-Link Bulb experiment
+//        final String inputPcapFile = path + "/training/tplink-bulb/wlan1/tplink-bulb.wlan1.local.pcap";
+//        // TP-Link Bulb PHONE signatures
+//        final String onSignatureFile = path + "/training/tplink-bulb/signatures/tplink-bulb-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/training/tplink-bulb/signatures/tplink-bulb-offSignature-phone-side.sig";
+        // TODO: EXPERIMENT - November 16, 2018
+        // TP-Link Bulb experiment
+//        final String inputPcapFile = path + "/experimental_result/standalone/tplink-bulb/wlan1/tplink-bulb.wlan1.local.pcap";
+//        final String inputPcapFile = path + "/experimental_result/standalone/tplink-bulb/eth0/tplink-bulb.eth0.local.pcap";
+//        final String inputPcapFile = path + "/experimental_result/smarthome/tplink-bulb/wlan1/tplink-bulb.wlan1.detection.pcap";
+////        final String inputPcapFile = path + "/experimental_result/smarthome/tplink-bulb/eth0/tplink-bulb.eth0.detection.pcap";
+//        // TP-Link Bulb PHONE signatures
+//        final String onSignatureFile = path + "/experimental_result/standalone/tplink-bulb/signatures/tplink-bulb-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/tplink-bulb/signatures/tplink-bulb-offSignature-phone-side.sig";
+
+        /*
+        // WeMo Plug experiment
+        final String inputPcapFile = path + "/training/wemo-plug/wlan1/wemo-plug.wlan1.local.pcap";
+        // WeMo Plug PHONE signatures
+        final String onSignatureFile = path + "/training/wemo-plug/signatures/wemo-plug-onSignature-device-side.sig";
+        final String offSignatureFile = path + "/training/wemo-plug/signatures/wemo-plug-offSignature-device-side.sig";
+        */
+        // TODO: EXPERIMENT - November 20, 2018
+        // WeMo Plug experiment
+//        final String inputPcapFile = path + "/experimental_result/standalone/wemo-plug/wlan1/wemo-plug.wlan1.local.pcap";
+//        final String inputPcapFile = path + "/experimental_result/standalone/wemo-plug/eth0/wemo-plug.eth0.local.pcap";
+        // TODO: WE HAVE 4 ADDITIONAL EVENTS (TRIGGERED MANUALLY), SO WE JUST IGNORE THEM BECAUSE THEY HAPPENED BEFORE
+        // TODO: THE ACTUAL TRIGGERS
+//        final String inputPcapFile = path + "/experimental_result/smarthome/wemo-plug/wlan1/wemo-plug.wlan1.detection.pcap";
+////        final String inputPcapFile = path + "/experimental_result/smarthome/wemo-plug/eth0/wemo-plug.eth0.detection.pcap";
+//        // WeMo Plug PHONE signatures
+//        final String onSignatureFile = path + "/experimental_result/standalone/wemo-plug/signatures/wemo-plug-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/wemo-plug/signatures/wemo-plug-offSignature-phone-side.sig";
+
+        /*
+        // WeMo Insight Plug experiment
+        final String inputPcapFile = path + "/training/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.local.pcap";
+        // WeMo Insight Plug PHONE signatures
+        final String onSignatureFile = path + "/training/wemo-insight-plug/signatures/wemo-insight-plug-onSignature-device-side.sig";
+        final String offSignatureFile = path + "/training/wemo-insight-plug/signatures/wemo-insight-plug-offSignature-device-side.sig";
+        */
+        // TODO: EXPERIMENT - November 21, 2018
+        // WeMo Insight Plug experiment
+//        final String inputPcapFile = path + "/experimental_result/standalone/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.local.pcap";
+//        final String inputPcapFile = path + "/experimental_result/standalone/wemo-insight-plug/eth0/wemo-insight-plug.eth0.local.pcap";
+        // TODO: WE HAVE 1 ADDITIONAL EVENT (FROM WEMO PLUG)
+//        final String inputPcapFile = path + "/experimental_result/smarthome/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.detection.pcap";
+//        final String inputPcapFile = path + "/experimental_result/smarthome/wemo-insight-plug/eth0/wemo-insight-plug.eth0.detection.pcap";
+        // WeMo Insight Plug PHONE signatures
+//        final String onSignatureFile = path + "/experimental_result/standalone/wemo-insight-plug/signatures/wemo-insight-plug-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/wemo-insight-plug/signatures/wemo-insight-plug-offSignature-phone-side.sig";
+
+
+        // Kwikset Doorlock Sep 12 experiment
+//        final String inputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset3.wlan1.local.pcap";
+//        // Kwikset Doorlock PHONE signatures
+//        final String onSignatureFile = path + "/2018-08/kwikset-doorlock/onSignature-Kwikset-Doorlock-phone.sig";
+//        final String offSignatureFile = path + "/2018-08/kwikset-doorlock/offSignature-Kwikset-Doorlock-phone.sig";
+        // TODO: EXPERIMENT - November 10, 2018
+        // Kwikset Door lock experiment
+//        final String inputPcapFile = path + "/experimental_result/standalone/kwikset-doorlock/wlan1/kwikset-doorlock.wlan1.local.pcap";
+//        //final String inputPcapFile = path + "/experimental_result/smarthome/kwikset-doorlock/wlan1/kwikset-doorlock.wlan1.detection.pcap";
+//        final String inputPcapFile = path + "/experimental_result/smarthome/kwikset-doorlock/eth0/kwikset-doorlock.eth0.detection.pcap";
+////        // Kwikset Door lock PHONE signatures
+//        final String onSignatureFile = path + "/experimental_result/standalone/kwikset-doorlock/signatures/kwikset-doorlock-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/experimental_result/standalone/kwikset-doorlock/signatures/kwikset-doorlock-offSignature-phone-side.sig";
+//        final String onSignatureFile = path + "/training/signatures/kwikset-doorlock/kwikset-doorlock-onSignature-phone-side.sig";
+//        final String offSignatureFile = path + "/training/signatures/kwikset-doorlock/kwikset-doorlock-offSignature-phone-side.sig";
+
+
+
+        // D-Link Siren experiment
+//        final String inputPcapFile = path + "/2018-08/dlink-siren/dlink-siren.wlan1.local.pcap";
+        // D-Link Siren DEVICE signatures
+        //final String onSignatureFile = path + "/2018-08/dlink-siren/onSignature-DLink-Siren-device.sig";
+        //final String offSignatureFile = path + "/2018-08/dlink-siren/offSignature-DLink-Siren-device.sig";
+        // D-Link Siren PHONE signatures
+//        final String onSignatureFile = path + "/2018-08/dlink-siren/onSignature-DLink-Siren-phone.sig";
+//        final String offSignatureFile = path + "/2018-08/dlink-siren/offSignature-DLink-Siren-phone.sig";
+
+
+        // Output file names used (to make it easy to catch if one forgets to change them)
+        System.out.println("ON signature file in use is " + onSignatureFile);
+        System.out.println("OFF signature file in use is " + offSignatureFile);
+        System.out.println("PCAP file that is the target of detection is " + inputPcapFile);
+
+        List<List<List<PcapPacket>>> onSignature = PrintUtils.deserializeSignatureFromFile(onSignatureFile);
+        List<List<List<PcapPacket>>> offSignature = PrintUtils.deserializeSignatureFromFile(offSignatureFile);
+
+        // LAN
+//        SignatureDetector onDetector = new SignatureDetector(onSignature, null);
+//        SignatureDetector offDetector = new SignatureDetector(offSignature, null);
+        // WAN
+        SignatureDetector onDetector = new SignatureDetector(onSignature, "128.195.205.105");
+        SignatureDetector offDetector = new SignatureDetector(offSignature, "128.195.205.105");
+
+        final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofLocalizedDateTime(FormatStyle.MEDIUM).
+                withLocale(Locale.US).withZone(ZoneId.of("America/Los_Angeles"));
+
+        // Outputs information about a detected event to std.out
+        final Consumer<UserAction> outputter = ua -> {
+            String eventDescription;
+            switch (ua.getType()) {
+                case TOGGLE_ON:
+                    eventDescription = "ON";
+                    break;
+                case TOGGLE_OFF:
+                    eventDescription = "OFF";
+                    break;
+                default:
+                    throw new AssertionError("unhandled event type");
+            }
+            //String output = String.format("[ !!! %s SIGNATURE DETECTED at %s !!! ]",
+            //      eventDescription, dateTimeFormatter.format(ua.getTimestamp()));
+            String output = String.format("%s",
+                    dateTimeFormatter.format(ua.getTimestamp()));
+            System.out.println(output);
+        };
+
+        // Let's create observers that construct a UserAction representing the detected event.
+        final List<UserAction> detectedEvents = new ArrayList<>();
+        onDetector.addObserver((searched, match) -> {
+            PcapPacket firstPkt = match.get(0).get(0);
+            detectedEvents.add(new UserAction(UserAction.Type.TOGGLE_ON, firstPkt.getTimestamp()));
+        });
+        offDetector.addObserver((searched, match) -> {
+            PcapPacket firstPkt = match.get(0).get(0);
+            detectedEvents.add(new UserAction(UserAction.Type.TOGGLE_OFF, firstPkt.getTimestamp()));
+        });
+
+        PcapHandle handle;
+        try {
+            handle = Pcaps.openOffline(inputPcapFile, PcapHandle.TimestampPrecision.NANO);
+        } catch (PcapNativeException pne) {
+            handle = Pcaps.openOffline(inputPcapFile);
+        }
+        PcapHandleReader reader = new PcapHandleReader(handle, p -> true, onDetector, offDetector);
+        reader.readFromHandle();
+
+        // TODO: need a better way of triggering detection than this...
+        onDetector.mClusterMatchers.forEach(cm -> cm.performDetection());
+        offDetector.mClusterMatchers.forEach(cm -> cm.performDetection());
+
+        // Sort the list of detected events by timestamp to make it easier to compare it line-by-line with the trigger
+        // times file.
+        Collections.sort(detectedEvents, Comparator.comparing(UserAction::getTimestamp));
+
+        // Output the detected events
+        detectedEvents.forEach(outputter);
+
+        System.out.println("Number of detected events of type " + UserAction.Type.TOGGLE_ON + ": " +
+                detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_ON).count());
+        System.out.println("Number of detected events of type " + UserAction.Type.TOGGLE_OFF + ": " +
+                detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_OFF).count());
+
+
+        // TODO: Temporary clean up until we clean the pipeline
+//        List<UserAction> cleanedDetectedEvents = SignatureDetector.removeDuplicates(detectedEvents);
+//        cleanedDetectedEvents.forEach(outputter);
+    }
+
+    /**
+     * The signature that this {@link SignatureDetector} is searching for.
+     */
+    private final List<List<List<PcapPacket>>> mSignature;
+
+    /**
+     * The {@link Layer3ClusterMatcher}s in charge of detecting each individual sequence of packets that together make up the
+     * the signature.
+     */
+    private final List<Layer3ClusterMatcher> mClusterMatchers;
+
+    /**
+     * For each {@code i} ({@code i >= 0 && i < pendingMatches.length}), {@code pendingMatches[i]} holds the matches
+     * found by the {@link Layer3ClusterMatcher} at {@code mClusterMatchers.get(i)} that have yet to be "consumed", i.e.,
+     * have yet to be included in a signature detected by this {@link SignatureDetector} (a signature can be encompassed
+     * of multiple packet sequences occurring shortly after one another on multiple connections).
+     */
+    private final List<List<PcapPacket>>[] pendingMatches;
+
+    /**
+     * Maps a {@link Layer3ClusterMatcher} to its corresponding index in {@link #pendingMatches}.
+     */
+    private final Map<Layer3ClusterMatcher, Integer> mClusterMatcherIds;
+
+    private final List<SignatureDetectionObserver> mObservers = new ArrayList<>();
+
+    /**
+     * Remove duplicates in {@code List} of {@code UserAction} objects. We need to clean this up for user actions
+     * that appear multiple times.
+     * TODO: This static method is probably just for temporary and we could get rid of this after we clean up
+     * TODO:    the pipeline
+     *
+     * @param listUserAction A {@link List} of {@code UserAction}.
+     *
+     */
+    public static List<UserAction> removeDuplicates(List<UserAction> listUserAction) {
+
+        // Iterate and check for duplicates (check timestamps)
+        Set<Long> epochSecondSet = new HashSet<>();
+        // Create a target list for cleaned up list
+        List<UserAction> listUserActionClean = new ArrayList<>();
+        for(UserAction userAction : listUserAction) {
+            // Don't insert if any duplicate is found
+            if(!epochSecondSet.contains(userAction.getTimestamp().getEpochSecond())) {
+                listUserActionClean.add(userAction);
+                epochSecondSet.add(userAction.getTimestamp().getEpochSecond());
+            }
+        }
+        return listUserActionClean;
+    }
+
+    public SignatureDetector(List<List<List<PcapPacket>>> searchedSignature, String routerWanIp) {
+        // note: doesn't protect inner lists from changes :'(
+        mSignature = Collections.unmodifiableList(searchedSignature);
+        // Generate corresponding/appropriate ClusterMatchers based on the provided signature
+        List<Layer3ClusterMatcher> clusterMatchers = new ArrayList<>();
+        for (List<List<PcapPacket>> cluster : mSignature) {
+            clusterMatchers.add(new Layer3ClusterMatcher(cluster, routerWanIp, this));
+        }
+        mClusterMatchers = Collections.unmodifiableList(clusterMatchers);
+
+        // < exploratory >
+        pendingMatches = new List[mClusterMatchers.size()];
+        for (int i = 0; i < pendingMatches.length; i++) {
+            pendingMatches[i] = new ArrayList<>();
+        }
+        Map<Layer3ClusterMatcher, Integer> clusterMatcherIds = new HashMap<>();
+        for (int i = 0; i < mClusterMatchers.size(); i++) {
+            clusterMatcherIds.put(mClusterMatchers.get(i), i);
+        }
+        mClusterMatcherIds = Collections.unmodifiableMap(clusterMatcherIds);
+    }
+
+    public void addObserver(SignatureDetectionObserver observer) {
+        mObservers.add(observer);
+    }
+
+    public boolean removeObserver(SignatureDetectionObserver observer) {
+        return mObservers.remove(observer);
+    }
+
+    @Override
+    public void gotPacket(PcapPacket packet) {
+        // simply delegate packet reception to all ClusterMatchers.
+        mClusterMatchers.forEach(cm -> cm.gotPacket(packet));
+    }
+
+    @Override
+    public void onMatch(Layer3ClusterMatcher clusterMatcher, List<PcapPacket> match) {
+        // Add the match at the corresponding index
+        pendingMatches[mClusterMatcherIds.get(clusterMatcher)].add(match);
+        checkSignatureMatch();
+    }
+
+    private void checkSignatureMatch() {
+        // << Graph-based approach using Balint's idea. >>
+        // This implementation assumes that the packets in the inner lists (the sequences) are ordered by asc timestamp.
+
+        // There cannot be a signature match until each Layer3ClusterMatcher has found a match of its respective sequence.
+        if (Arrays.stream(pendingMatches).noneMatch(l -> l.isEmpty())) {
+            // Construct the DAG
+            final SimpleDirectedWeightedGraph<Vertex, DefaultWeightedEdge> graph =
+                    new SimpleDirectedWeightedGraph<>(DefaultWeightedEdge.class);
+            // Add a vertex for each match found by all ClusterMatchers
+            // And maintain an array to keep track of what cluster matcher each vertex corresponds to
+            final List<Vertex>[] vertices = new List[pendingMatches.length];
+            for (int i = 0; i < pendingMatches.length; i++) {
+                vertices[i] = new ArrayList<>();
+                for (List<PcapPacket> sequence : pendingMatches[i]) {
+                    Vertex v = new Vertex(sequence);
+                    vertices[i].add(v); // retain reference for later when we are to add edges
+                    graph.addVertex(v); // add to vertex to graph
+                }
+            }
+            // Add dummy source and sink vertices to facilitate search.
+            final Vertex source = new Vertex(null);
+            final Vertex sink = new Vertex(null);
+            graph.addVertex(source);
+            graph.addVertex(sink);
+            // The source is connected to all vertices that wrap the sequences detected by Layer3ClusterMatcher at index 0.
+            // Note: zero cost edges as this is just a dummy link to facilitate search from a common start node.
+            for (Vertex v : vertices[0]) {
+                DefaultWeightedEdge edge = graph.addEdge(source, v);
+                graph.setEdgeWeight(edge, 0.0);
+            }
+            // Similarly, all vertices that wrap the sequences detected by the last Layer3ClusterMatcher of the signature
+            // are connected to the sink node.
+            for (Vertex v : vertices[vertices.length-1]) {
+                DefaultWeightedEdge edge = graph.addEdge(v, sink);
+                graph.setEdgeWeight(edge, 0.0);
+            }
+            // Now link sequences detected by Layer3ClusterMatcher at index i to sequences detected by Layer3ClusterMatcher at index
+            // i+1 if they obey the timestamp constraint (i.e., that the latter is later in time than the former).
+            for (int i = 0; i < vertices.length; i++) {
+                int j = i + 1;
+                if (j < vertices.length) {
+                    for (Vertex iv : vertices[i]) {
+                        PcapPacket ivLast = iv.sequence.get(iv.sequence.size()-1);
+                        for (Vertex jv : vertices[j]) {
+                            PcapPacket jvFirst = jv.sequence.get(jv.sequence.size()-1);
+                            if (ivLast.getTimestamp().isBefore(jvFirst.getTimestamp())) {
+                                DefaultWeightedEdge edge = graph.addEdge(iv, jv);
+                                // The weight is the duration of the i'th sequence plus the duration between the i'th
+                                // and i+1'th sequence.
+                                Duration d = Duration.
+                                        between(iv.sequence.get(0).getTimestamp(), jvFirst.getTimestamp());
+                                // Unfortunately weights are double values, so must convert from long to double.
+                                // TODO: need nano second precision? If so, use d.toNanos().
+                                // TODO: risk of overflow when converting from long to double..?
+                                graph.setEdgeWeight(edge, Long.valueOf(d.toMillis()).doubleValue());
+                            }
+                            // Alternative version if we cannot assume that sequences are ordered by timestamp:
+//                            if (iv.sequence.stream().max(Comparator.comparing(PcapPacket::getTimestamp)).get()
+//                                    .getTimestamp().isBefore(jv.sequence.stream().min(
+//                                            Comparator.comparing(PcapPacket::getTimestamp)).get().getTimestamp())) {
+//
+//                            }
+                        }
+                    }
+                }
+            }
+            // Graph construction complete, run shortest-path to find a (potential) signature match.
+            DijkstraShortestPath<Vertex, DefaultWeightedEdge> dijkstra = new DijkstraShortestPath<>(graph);
+            GraphPath<Vertex, DefaultWeightedEdge> shortestPath = dijkstra.getPath(source, sink);
+            if (shortestPath != null) {
+                // The total weight is the duration between the first packet of the first sequence and the last packet
+                // of the last sequence, so we simply have to compare the weight against the timeframe that we allow
+                // the signature to span. For now we just use the inclusion window we defined for training purposes.
+                // Note however, that we must convert back from double to long as the weight is stored as a double in
+                // JGraphT's API.
+                if (((long)shortestPath.getWeight()) < TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS) {
+                    // There's a signature match!
+                    // Extract the match from the vertices
+                    List<List<PcapPacket>> signatureMatch = new ArrayList<>();
+                    for(Vertex v : shortestPath.getVertexList()) {
+                        if (v == source || v == sink) {
+                            // Skip the dummy source and sink nodes.
+                            continue;
+                        }
+                        signatureMatch.add(v.sequence);
+                        // As there is a one-to-one correspondence between vertices[] and pendingMatches[], we know that
+                        // the sequence we've "consumed" for index i of the matched signature is also at index i in
+                        // pendingMatches. We must remove it from pendingMatches so that we don't use it to construct
+                        // another signature match in a later call.
+                        pendingMatches[signatureMatch.size()-1].remove(v.sequence);
+                    }
+                    // Declare success: notify observers
+                    mObservers.forEach(obs -> obs.onSignatureDetected(mSignature,
+                            Collections.unmodifiableList(signatureMatch)));
+                }
+            }
+        }
+    }
+
+    /**
+     * Used for registering for notifications of signatures detected by a {@link SignatureDetector}.
+     */
+    interface SignatureDetectionObserver {
+
+        /**
+         * Invoked when the {@link SignatureDetector} detects the presence of a signature in the traffic that it's
+         * examining.
+         * @param searchedSignature The signature that the {@link SignatureDetector} reporting the match is searching
+         *                          for.
+         * @param matchingTraffic The actual traffic trace that matches the searched signature.
+         */
+        void onSignatureDetected(List<List<List<PcapPacket>>> searchedSignature,
+                                 List<List<PcapPacket>> matchingTraffic);
+    }
+
+    /**
+     * Encapsulates a {@code List<PcapPacket>} so as to allow the list to be used as a vertex in a graph while avoiding
+     * the expensive {@link AbstractList#equals(Object)} calls when adding vertices to the graph.
+     * Using this wrapper makes the incurred {@code equals(Object)} calls delegate to {@link Object#equals(Object)}
+     * instead of {@link AbstractList#equals(Object)}. The net effect is a faster implementation, but the graph will not
+     * recognize two lists that contain the same items--from a value and not reference point of view--as the same
+     * vertex. However, this is fine for our purposes -- in fact restricting it to reference equality seems more
+     * appropriate.
+     */
+    private static class Vertex {
+        private final List<PcapPacket> sequence;
+        private Vertex(List<PcapPacket> wrappedSequence) {
+            sequence = wrappedSequence;
+        }
+    }
+}
index 9cecf56bc599938b6d682d8679ddcbcd4b39a88d..060387a0a93edbfc212add3dcb9bf7793a088a3a 100644 (file)
@@ -1,7 +1,7 @@
 package edu.uci.iotproject.evaluation;
 
-import edu.uci.iotproject.Conversation;
-import edu.uci.iotproject.TcpReassembler;
+import edu.uci.iotproject.trafficreassembly.layer3.Conversation;
+import edu.uci.iotproject.trafficreassembly.layer3.TcpReassembler;
 import edu.uci.iotproject.io.PcapHandleReader;
 import edu.uci.iotproject.util.PrintUtils;
 import org.pcap4j.core.*;
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2Flow.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2Flow.java
new file mode 100644 (file)
index 0000000..2d804bd
--- /dev/null
@@ -0,0 +1,95 @@
+package edu.uci.iotproject.trafficreassembly.layer2;
+
+import org.pcap4j.core.PcapPacket;
+import org.pcap4j.packet.EthernetPacket;
+import org.pcap4j.util.MacAddress;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Models a layer 2 flow: groups packets exchanged between two specific endpoints (MAC addresses).
+ *
+ * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
+ * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
+ */
+public class Layer2Flow {
+
+    /**
+     * The first endpoint of this layer 2 flow.
+     */
+    private final MacAddress mEndpoint1;
+
+    /**
+     * The second endpoint of this layer 2 flow.
+     */
+    private final MacAddress mEndpoint2;
+
+    /**
+     * Clients observing for changes to this layer 2 flow.
+     */
+    private final List<Layer2FlowObserver> mFlowObservers = new ArrayList<>();
+
+    public Layer2Flow(MacAddress endpoint1, MacAddress endpoint2) {
+        mEndpoint1 = endpoint1;
+        mEndpoint2 = endpoint2;
+    }
+
+    /**
+     * Register as an observer of this flow.
+     * @param observer The client that is to be notified whenever this flow changes (has new packets added).
+     */
+    public void addFlowObserver(Layer2FlowObserver observer) {
+        mFlowObservers.add(observer);
+    }
+
+    /**
+     * Deregister as an observer of this flow.
+     * @param observer The client that no longer wishes to be notified whenever this flow changes.
+     */
+    public void removeFlowObserver(Layer2FlowObserver observer) {
+        mFlowObservers.remove(observer);
+    }
+
+    /**
+     * The packets in the flow.
+     */
+    private final List<PcapPacket> mPackets = new ArrayList<>();
+
+    /**
+     * Add a packet to this flow.
+     * @param packet The packet that is to be added to the flow.
+     */
+    public void addPacket(PcapPacket packet) {
+        verifyAddresses(packet);
+        mPackets.add(packet);
+        // Notify flow observers of the new packet
+        mFlowObservers.forEach(o -> o.onNewPacket(this, packet));
+    }
+
+    /**
+     * Get the packets pertaining to this flow.
+     * @return The packets pertaining to this flow.
+     */
+    public List<PcapPacket> getPackets() {
+        return Collections.unmodifiableList(mPackets);
+    }
+
+    /**
+     * Verify that a packet pertains to this flow.
+     * @param packet The packet that is to be verified.
+     */
+    private void verifyAddresses(PcapPacket packet) {
+        EthernetPacket ethPkt = packet.get(EthernetPacket.class);
+        MacAddress srcAddr = ethPkt.getHeader().getSrcAddr();
+        MacAddress dstAddr = ethPkt.getHeader().getDstAddr();
+        if ((mEndpoint1.equals(srcAddr) && mEndpoint2.equals(dstAddr)) ||
+                (mEndpoint1.equals(dstAddr) && mEndpoint2.equals(srcAddr))) {
+            // All is good.
+            return;
+        }
+        throw new IllegalArgumentException("Mismatch in MACs: packet does not pertain to this flow");
+    }
+
+}
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowObserver.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowObserver.java
new file mode 100644 (file)
index 0000000..e1648ba
--- /dev/null
@@ -0,0 +1,20 @@
+package edu.uci.iotproject.trafficreassembly.layer2;
+
+import org.pcap4j.core.PcapPacket;
+
+/**
+ * Interface for observing a {@link Layer2Flow}.
+ *
+ * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
+ * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
+ */
+public interface Layer2FlowObserver {
+
+    /**
+     * Invoked when a new packet is added to the observed flow.
+     * @param flow The observed flow.
+     * @param newPacket The packet that was added to the flow.
+     */
+    void onNewPacket(Layer2Flow flow, PcapPacket newPacket);
+
+}
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowReassembler.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowReassembler.java
new file mode 100644 (file)
index 0000000..e7b7304
--- /dev/null
@@ -0,0 +1,86 @@
+package edu.uci.iotproject.trafficreassembly.layer2;
+
+import edu.uci.iotproject.trafficreassembly.layer2.Layer2Flow;
+import edu.uci.iotproject.trafficreassembly.layer2.Layer2FlowReassemblerObserver;
+import org.pcap4j.core.PacketListener;
+import org.pcap4j.core.PcapPacket;
+import org.pcap4j.packet.EthernetPacket;
+import org.pcap4j.util.MacAddress;
+
+import java.util.*;
+
+/**
+ * Reassembles traffic flows at layer 2, i.e., for each combination of hosts, creates a list of packets exchanged
+ * between said hosts.
+ *
+ * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
+ * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
+ */
+public class Layer2FlowReassembler implements PacketListener {
+
+    /**
+     * Maps a pair of MAC addresses to the packets exchanged between the two hosts.
+     * The key is the concatenation of the two MAC addresses in hex string format, where the lexicographically smaller
+     * MAC is at the front of the string.
+     */
+    private final Map<String, Layer2Flow> mFlows = new HashMap<>();
+
+    private final List<Layer2FlowReassemblerObserver> mObservers = new ArrayList<>();
+
+    @Override
+    public void gotPacket(PcapPacket packet) {
+        // TODO: update to 802.11 packet...?
+        EthernetPacket ethPkt = packet.get(EthernetPacket.class);
+
+        MacAddress srcAddr = ethPkt.getHeader().getSrcAddr();
+        MacAddress dstAddr = ethPkt.getHeader().getDstAddr();
+
+        String key = keyFromAddresses(srcAddr, dstAddr);
+        // Create a new list if this pair of MAC addresses where not previously encountered and add packet to that list,
+        // or simply add to an existing list if one is present.
+        mFlows.computeIfAbsent(key, k -> {
+            Layer2Flow newFlow = new Layer2Flow(srcAddr, dstAddr);
+            // Inform observers of the new flow
+            mObservers.forEach(o -> o.onNewFlow(this, newFlow));
+            return newFlow;
+        }).addPacket(packet);
+    }
+
+    public void addObserver(Layer2FlowReassemblerObserver observer) {
+        mObservers.add(observer);
+    }
+
+    public void removeObserver(Layer2FlowReassemblerObserver observer) {
+        mObservers.remove(observer);
+    }
+
+    /**
+     * Get the traffic flow between two local endpoints ({@link MacAddress}es).
+     * @param addr1 The first endpoint.
+     * @param addr2 The second endpoint
+     * @return The traffic exchanged between the two endpoints.
+     */
+    public Layer2Flow getFlowForAddresses(MacAddress addr1, MacAddress addr2) {
+        return mFlows.get(keyFromAddresses(addr1, addr2));
+    }
+
+    /**
+     * Get all traffic flows, i.e., a traffic flow for each unique pair of endpoints (MAC addresses).
+     * @return All traffic flows.
+     */
+    public Collection<Layer2Flow> getFlows() {
+        return mFlows.values();
+    }
+
+    /**
+     * Given two {@link MacAddress}es, generates the corresponding key string used in {@link #mFlows}.
+     * @param addr1 The first address.
+     * @param addr2 The second address.
+     * @return the key string used in {@link #mFlows} corresponding to the two addresses.
+     */
+    private String keyFromAddresses(MacAddress addr1, MacAddress addr2) {
+        String addr1Str = addr1.toString();
+        String addr2Str = addr2.toString();
+        return addr1Str.compareTo(addr2Str) < 0 ? addr1Str + addr2Str : addr2Str + addr1Str;
+    }
+}
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowReassemblerObserver.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowReassemblerObserver.java
new file mode 100644 (file)
index 0000000..a71cd19
--- /dev/null
@@ -0,0 +1,19 @@
+package edu.uci.iotproject.trafficreassembly.layer2;
+
+/**
+ * For observing a {@link Layer2FlowReassembler}.
+ *
+ * @author Janus Varmarken
+ */
+public interface Layer2FlowReassemblerObserver {
+
+    /**
+     * Invoked when when a {@link Layer2FlowReassembler} detects a new flow (i.e., when it encounters traffic between two
+     * MAC addresses that has not previously communicated in the traffic trace).
+     *
+     * @param reassembler The {@link Layer2FlowReassembler} that detected the new flow.
+     * @param newFlow The new flow.
+     */
+    void onNewFlow(Layer2FlowReassembler reassembler, Layer2Flow newFlow);
+
+}
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/Conversation.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/Conversation.java
new file mode 100644 (file)
index 0000000..e89e81b
--- /dev/null
@@ -0,0 +1,585 @@
+package edu.uci.iotproject.trafficreassembly.layer3;
+
+import edu.uci.iotproject.analysis.TcpConversationUtils;
+import edu.uci.iotproject.util.PcapPacketUtils;
+import org.pcap4j.core.PcapPacket;
+import org.pcap4j.packet.IpV4Packet;
+import org.pcap4j.packet.Packet;
+import org.pcap4j.packet.TcpPacket;
+
+import java.util.*;
+
+/**
+ * Models a (TCP) conversation/connection/session/flow (packet's belonging to the same session between a client and a
+ * server).
+ * Holds a list of {@link PcapPacket}s identified as pertaining to the flow. Note that this list is <em>not</em>
+ * considered when determining equality of two {@code Conversation} instances in order to allow for a
+ * {@code Conversation} to function as a key in data structures such as {@link java.util.Map} and {@link java.util.Set}.
+ * See {@link #equals(Object)} for the definition of equality.
+ *
+ * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
+ * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
+ */
+public class Conversation {
+
+    /* Begin instance properties */
+    /**
+     * The IP of the host that is considered the client (i.e. the host that initiates the conversation)
+     * in this conversation.
+     */
+    private final String mClientIp;
+
+    /**
+     * The port number used by the host that is considered the client in this conversation.
+     */
+    private final int mClientPort;
+
+    /**
+     * The IP of the host that is considered the server (i.e. is the responder) in this conversation.
+     */
+    private final String mServerIp;
+
+    /**
+     * The port number used by the server in this conversation.
+     */
+    private final int mServerPort;
+
+    /**
+     * The list of packets (with payload) pertaining to this conversation.
+     */
+    private final List<PcapPacket> mPackets;
+
+    /**
+     * If {@link #isTls()} is {@code true}, this list contains the subset of {@link #mPackets} which are TLS Application
+     * Data packets.
+     */
+    private final List<PcapPacket> mTlsApplicationDataPackets;
+
+    /**
+     * Contains the sequence numbers used thus far by the host that is considered the <em>client</em> in this
+     * {@code Conversation}.
+     * Used for filtering out retransmissions.
+     */
+    private final Set<Integer> mSeqNumbersClient;
+
+    /**
+     * Contains the sequence numbers used thus far by the host that is considered the <em>server</em> in this
+     * {@code Conversation}.
+     * Used for filtering out retransmissions.
+     */
+    private final Set<Integer> mSeqNumbersSrv;
+
+    /**
+     * List of SYN packets pertaining to this conversation.
+     */
+    private final List<PcapPacket> mSynPackets;
+
+    /**
+     * List of pairs FINs and their corresponding ACKs associated with this conversation.
+     */
+    private final List<FinAckPair> mFinPackets;
+
+    /**
+     * List of RST packets associated with this conversation.
+     */
+    private final List<PcapPacket> mRstPackets;
+
+    /**
+     * Boolean to mark the packet as Application Data based on the previous packet that reaches MTU
+     */
+    private boolean mApplicationData;
+    /* End instance properties */
+
+    /**
+     * Factory method for creating a {@code Conversation} from a {@link PcapPacket}.
+     * @param pcapPacket The {@code PcapPacket} that wraps a TCP segment for which a {@code Conversation} is to be initiated.
+     * @param clientIsSrc If {@code true}, the source address and source port found in the IP datagram and TCP segment
+     *                    wrapped in the {@code PcapPacket} are regarded as pertaining to the client, and the destination
+     *                    address and destination port are regarded as pertaining to the server---and vice versa if set
+     *                    to {@code false}.
+     * @return A {@code Conversation} initiated with ip:port for client and server according to the direction of the packet.
+     */
+    public static Conversation fromPcapPacket(PcapPacket pcapPacket, boolean clientIsSrc) {
+        IpV4Packet ipPacket = pcapPacket.get(IpV4Packet.class);
+        TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class);
+        String clientIp = clientIsSrc ? ipPacket.getHeader().getSrcAddr().getHostAddress() :
+                ipPacket.getHeader().getDstAddr().getHostAddress();
+        String srvIp = clientIsSrc ? ipPacket.getHeader().getDstAddr().getHostAddress() :
+                ipPacket.getHeader().getSrcAddr().getHostAddress();
+        int clientPort = clientIsSrc ? tcpPacket.getHeader().getSrcPort().valueAsInt() :
+                tcpPacket.getHeader().getDstPort().valueAsInt();
+        int srvPort = clientIsSrc ? tcpPacket.getHeader().getDstPort().valueAsInt() :
+                tcpPacket.getHeader().getSrcPort().valueAsInt();
+        return new Conversation(clientIp, clientPort, srvIp, srvPort);
+    }
+
+    /**
+     * Constructs a new {@code Conversation}.
+     * @param clientIp The IP of the host that is considered the client (i.e. the host that initiates the conversation)
+     *                 in the conversation.
+     * @param clientPort The port number used by the client for the conversation.
+     * @param serverIp The IP of the host that is considered the server (i.e. is the responder) in the conversation.
+     * @param serverPort The port number used by the server for the conversation.
+     */
+    public Conversation(String clientIp, int clientPort, String serverIp, int serverPort) {
+        this.mClientIp = clientIp;
+        this.mClientPort = clientPort;
+        this.mServerIp = serverIp;
+        this.mServerPort = serverPort;
+        this.mPackets = new ArrayList<>();
+        this.mTlsApplicationDataPackets = new ArrayList<>();
+        this.mSeqNumbersClient = new HashSet<>();
+        this.mSeqNumbersSrv = new HashSet<>();
+        this.mSynPackets = new ArrayList<>();
+        this.mFinPackets = new ArrayList<>();
+        this.mRstPackets = new ArrayList<>();
+        this.mApplicationData = false;
+    }
+
+    /**
+     * Add a packet to the list of packets associated with this conversation.
+     * @param packet The packet that is to be added to (associated with) this conversation.
+     * @param ignoreRetransmissions Boolean value indicating if retransmissions should be ignored.
+     *                              If set to {@code true}, {@code packet} will <em>not</em> be added to the
+     *                              internal list of packets pertaining to this {@code Conversation}
+     *                              <em>iff</em> the sequence number of {@code packet} was already
+     *                              seen in a previous packet.
+     */
+    public void addPacket(PcapPacket packet, boolean ignoreRetransmissions) {
+        // Precondition: verify that packet does indeed pertain to conversation.
+        onAddPrecondition(packet);
+        if (ignoreRetransmissions && isRetransmission(packet)) {
+            // Packet is a retransmission. Ignore it.
+            return;
+        }
+        // Select direction-dependent set of sequence numbers seen so far and update it with sequence number of new packet.
+        addSeqNumber(packet);
+        // Finally add packet to list of packets pertaining to this conversation.
+        mPackets.add(packet);
+        // Preserve order of packets in list: sort according to timestamp.
+        if (mPackets.size() > 1 &&
+                mPackets.get(mPackets.size()-1).getTimestamp().isBefore(mPackets.get(mPackets.size()-2).getTimestamp())) {
+            Collections.sort(mPackets, (o1, o2) -> {
+                if (o1.getTimestamp().isBefore(o2.getTimestamp())) { return -1; }
+                else if (o2.getTimestamp().isBefore(o1.getTimestamp())) { return 1; }
+                else { return 0; }
+            });
+        }
+        // If TLS, inspect packet to see if it's a TLS Application Data packet, and if so add it to the list of TLS
+        // Application Data packets.
+        if (isTls()) {
+            TcpPacket tcpPacket = packet.get(TcpPacket.class);
+            Packet tcpPayload = tcpPacket.getPayload();
+            if (tcpPayload == null) {
+                return;
+            }
+            byte[] rawPayload = tcpPayload.getRawData();
+            // The SSL record header is at the front of the payload and is 5 bytes long.
+            // The SSL record header type field (the first byte) is set to 23 if it is an Application Data packet.
+            if (rawPayload != null && rawPayload.length >= 5) {
+                if (rawPayload[0] == 23) {
+                    mTlsApplicationDataPackets.add(packet);
+                    // Consider the following packet a data packet if this packet's size == MTU size 1448
+                    if (rawPayload.length >= 1448)
+                        mApplicationData = true;
+                } else if (rawPayload[0] == 20) {
+                    // Do nothing for now - CHANGE_CIPHER_SPEC
+                } else if (rawPayload[0] == 21) {
+                    // Do nothing for now - ALERT
+                } else if (rawPayload[0] == 22) {
+                    // Do nothing for now - HANDSHAKE
+                } else {
+                    // If it is TLS with payload, but rawPayload[0] != 23
+                    if (mApplicationData == true) {
+                        // It is a continuation of the previous packet if the previous packet reaches MTU size 1448 and
+                        // it is not either type 20, 21, or 22
+                        mTlsApplicationDataPackets.add(packet);
+                        if (rawPayload.length < 1448)
+                            mApplicationData = false;
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * Get a list of packets pertaining to this {@code Conversation}.
+     * The returned list is a read-only list.
+     * @return the list of packets pertaining to this {@code Conversation}.
+     */
+    public List<PcapPacket> getPackets() {
+        // Return read-only view to prevent external code from manipulating internal state (preserve invariant).
+        return Collections.unmodifiableList(mPackets);
+    }
+
+    /**
+     * Records a TCP SYN packet as pertaining to this conversation (adds it to the the internal list).
+     * Attempts to add duplicate SYN packets will be ignored, and the caller is made aware of the attempt to add a
+     * duplicate by the return value being {@code false}.
+     *
+     * @param synPacket A {@link PcapPacket} wrapping a TCP SYN packet.
+     * @return {@code true} if the packet was successfully added to this {@code Conversation}, {@code false} otherwise.
+     */
+    public boolean addSynPacket(PcapPacket synPacket) {
+        onAddPrecondition(synPacket);
+        final IpV4Packet synPacketIpSection = synPacket.get(IpV4Packet.class);
+        final TcpPacket synPacketTcpSection = synPacket.get(TcpPacket.class);
+        if (synPacketTcpSection == null || !synPacketTcpSection.getHeader().getSyn()) {
+            throw new IllegalArgumentException("Not a SYN packet.");
+        }
+        // We are only interested in recording one copy of the two SYN packets (one SYN packet in each direction), i.e.,
+        // we want to discard retransmitted SYN packets.
+        if (mSynPackets.size() >= 2) {
+            return false;
+        }
+        // Check the set of recorded SYN packets to see if we have already recorded a SYN packet going in the same
+        // direction as the packet given in the argument.
+        boolean matchingPrevSyn = mSynPackets.stream().anyMatch(p -> {
+            IpV4Packet pIp = p.get(IpV4Packet.class);
+            TcpPacket pTcp = p.get(TcpPacket.class);
+            boolean srcAddrMatch = synPacketIpSection.getHeader().getSrcAddr().getHostAddress().
+                    equals(pIp.getHeader().getSrcAddr().getHostAddress());
+            boolean dstAddrMatch = synPacketIpSection.getHeader().getDstAddr().getHostAddress().
+                    equals(pIp.getHeader().getDstAddr().getHostAddress());
+            boolean srcPortMatch = synPacketTcpSection.getHeader().getSrcPort().valueAsInt() ==
+                    pTcp.getHeader().getSrcPort().valueAsInt();
+            boolean dstPortMatch = synPacketTcpSection.getHeader().getDstPort().valueAsInt() ==
+                    pTcp.getHeader().getDstPort().valueAsInt();
+            return srcAddrMatch && dstAddrMatch && srcPortMatch && dstPortMatch;
+        });
+        if (matchingPrevSyn) {
+            return false;
+        }
+        // Update direction-dependent set of sequence numbers and record/log packet.
+        addSeqNumber(synPacket);
+        return mSynPackets.add(synPacket);
+
+        /*
+        mSynPackets.stream().anyMatch(p -> {
+            IpV4Packet pIp = p.get(IpV4Packet.class);
+            TcpPacket pTcp = p.get(TcpPacket.class);
+            boolean srcAddrMatch = synPacketIpSection.getHeader().getSrcAddr().getHostAddress().
+                    equals(pIp.getHeader().getSrcAddr().getHostAddress());
+            boolean dstAddrMatch = synPacketIpSection.getHeader().getDstAddr().getHostAddress().
+                    equals(pIp.getHeader().getDstAddr().getHostAddress());
+            boolean srcPortMatch = synPacketTcpSection.getHeader().getSrcPort().valueAsInt() ==
+                    pTcp.getHeader().getSrcPort().valueAsInt();
+            boolean dstPortMatch = synPacketTcpSection.getHeader().getDstPort().value() ==
+                    pTcp.getHeader().getDstPort().value();
+
+            boolean fourTupleMatch = srcAddrMatch && dstAddrMatch && srcPortMatch && dstPortMatch;
+
+            boolean seqNoMatch = synPacketTcpSection.getHeader().getSequenceNumber() ==
+                    pTcp.getHeader().getSequenceNumber();
+
+            if (fourTupleMatch && !seqNoMatch) {
+                // If the four tuple that identifies the conversation matches, but the sequence number is different,
+                // it means that this SYN packet is, in fact, an attempt to establish a **new** connection, and hence
+                // the given packet is NOT part of this conversation, even though the ip:port combinations are (by
+                // chance) selected such that they match this conversation.
+                throw new IllegalArgumentException("Attempt to add SYN packet that belongs to a different conversation " +
+                        "(which is identified by the same four tuple as this conversation)");
+            }
+            return fourTupleMatch && seqNoMatch;
+        });
+        */
+    }
+
+    /**
+     * Get a list of SYN packets pertaining to this {@code Conversation}.
+     * The returned list is a read-only list.
+     * @return the list of SYN packets pertaining to this {@code Conversation}.
+     */
+    public List<PcapPacket> getSynPackets() {
+        return Collections.unmodifiableList(mSynPackets);
+    }
+
+    /**
+     * Adds a TCP FIN packet to the list of TCP FIN packets associated with this conversation.
+     * @param finPacket The TCP FIN packet that is to be added to (associated with) this conversation.
+     */
+    public void addFinPacket(PcapPacket finPacket) {
+        // Precondition: verify that packet does indeed pertain to conversation.
+        onAddPrecondition(finPacket);
+        // TODO: should call addSeqNumber here?
+        addSeqNumber(finPacket);
+        mFinPackets.add(new FinAckPair(finPacket));
+    }
+
+    /**
+     * Attempt to ACK any FIN packets held by this conversation.
+     * @param ackPacket The ACK for a FIN previously added to this conversation.
+     */
+    public void attemptAcknowledgementOfFin(PcapPacket ackPacket) {
+        // Precondition: verify that the packet pertains to this conversation.
+        onAddPrecondition(ackPacket);
+        // Mark unack'ed FIN(s) that this ACK matches as ACK'ed (there might be more than one in case of retransmissions..?)
+        mFinPackets.replaceAll(finAckPair -> !finAckPair.isAcknowledged() && finAckPair.isCorrespondingAckPacket(ackPacket) ? new FinAckPair(finAckPair.getFinPacket(), ackPacket) : finAckPair);
+    }
+
+    /**
+     * Retrieves an unmodifiable view of the list of {@link FinAckPair}s associated with this {@code Conversation}.
+     * @return an unmodifiable view of the list of {@link FinAckPair}s associated with this {@code Conversation}.
+     */
+    public List<FinAckPair> getFinAckPairs() {
+        return Collections.unmodifiableList(mFinPackets);
+    }
+
+    /**
+     * Get if this {@code Conversation} is considered to have been gracefully shut down.
+     * A {@code Conversation} has been gracefully shut down if it contains a FIN+ACK pair for both directions
+     * (client to server, and server to client).
+     * @return {@code true} if the connection has been gracefully shut down, false otherwise.
+     */
+    public boolean isGracefullyShutdown() {
+        //  The conversation has been gracefully shut down if we have recorded a FIN from both the client and the server which have both been ack'ed.
+        return mFinPackets.stream().anyMatch(finAckPair -> finAckPair.isAcknowledged() && PcapPacketUtils.isSource(finAckPair.getFinPacket(), mClientIp, mClientPort)) &&
+                mFinPackets.stream().anyMatch(finAckPair -> finAckPair.isAcknowledged() && PcapPacketUtils.isSource(finAckPair.getFinPacket(), mServerIp, mServerPort));
+    }
+
+    /**
+     * Add a TCP segment for which the RST flag is set to this {@code Conversation}.
+     * @param packet A {@link PcapPacket} wrapping a TCP segment pertaining to this {@code Conversation} for which the
+     *               RST flag is set.
+     */
+    public void addRstPacket(PcapPacket packet) {
+        /*
+         * TODO:
+         * When now also keeping track of RST packets, should we also...?
+         * 1) Prevent later packets from being added once a RST segment has been added?
+         * 2) Extend 'isGracefullyShutdown()' to also consider RST segments, or add another method, 'isShutdown()' that
+         *    both considers FIN/ACK (graceful) as well as RST (abrupt/"ungraceful") shutdown?
+         * 3) Should it be impossible to associate more than one RST segment with each Conversation?
+         */
+        onAddPrecondition(packet);
+        TcpPacket tcpPacket = packet.get(TcpPacket.class);
+        if (tcpPacket == null || !tcpPacket.getHeader().getRst()) {
+            throw new IllegalArgumentException("not a RST packet");
+        }
+        mRstPackets.add(packet);
+    }
+
+    /**
+     * Get the TCP segments pertaining to this {@code Conversation} for which it was detected that the RST flag is set.
+     * @return the TCP segments pertaining to this {@code Conversation} for which it was detected that the RST flag is
+     *         set.
+     */
+    public List<PcapPacket> getRstPackets() {
+        return Collections.unmodifiableList(mRstPackets);
+    }
+
+    // =========================================================================================================
+    // We simply reuse equals and hashCode methods of String.class to be able to use this class as a key
+    // in a Map.
+
+    /**
+     * <em>Note:</em> currently, equality is determined based on pairwise equality of the elements of the four tuple
+     * ({@link #mClientIp}, {@link #mClientPort}, {@link #mServerIp}, {@link #mServerPort}) for {@code this} and
+     * {@code obj}.
+     * @param obj The object to test for equality with {@code this}.
+     * @return {@code true} if {@code obj} is considered equal to {@code this} based on the definition of equality given above.
+     */
+    @Override
+    public boolean equals(Object obj) {
+        return obj instanceof Conversation && this.toString().equals(obj.toString());
+    }
+
+    @Override
+    public int hashCode() {
+        return toString().hashCode();
+    }
+    // =========================================================================================================
+
+    @Override
+    public String toString() {
+        return String.format("%s:%d %s:%d", mClientIp, mClientPort, mServerIp, mServerPort);
+    }
+
+    /**
+     * Invoke to verify that the precondition holds when a caller attempts to add a packet to this {@code Conversation}.
+     * An {@link IllegalArgumentException} is thrown if the precondition is violated.
+     * @param packet the packet to be added to this {@code Conversation}
+     */
+    private void onAddPrecondition(PcapPacket packet) {
+        // Apply precondition to preserve class invariant: all packets in mPackets must match the 4 tuple that
+        // defines the conversation.
+        IpV4Packet ipPacket = Objects.requireNonNull(packet.get(IpV4Packet.class));
+        // For now we only support TCP flows.
+        TcpPacket tcpPacket = Objects.requireNonNull(packet.get(TcpPacket.class));
+        String ipSrc = ipPacket.getHeader().getSrcAddr().getHostAddress();
+        String ipDst = ipPacket.getHeader().getDstAddr().getHostAddress();
+        int srcPort = tcpPacket.getHeader().getSrcPort().valueAsInt();
+        int dstPort = tcpPacket.getHeader().getDstPort().valueAsInt();
+        String clientIp, serverIp;
+        int clientPort, serverPort;
+        if (ipSrc.equals(mClientIp)) {
+            clientIp = ipSrc;
+            clientPort = srcPort;
+            serverIp = ipDst;
+            serverPort = dstPort;
+        } else {
+            clientIp = ipDst;
+            clientPort = dstPort;
+            serverIp = ipSrc;
+            serverPort = srcPort;
+        }
+        if (!(clientIp.equals(mClientIp) && clientPort == mClientPort &&
+                serverIp.equals(mServerIp) && serverPort == mServerPort)) {
+            throw new IllegalArgumentException(
+                    String.format("Attempt to add packet that does not pertain to %s",
+                            Conversation.class.getSimpleName()));
+        }
+    }
+
+    /**
+     * <p>
+     *      Determines if the TCP packet contained in {@code packet} is a retransmission of a previously seen (logged)
+     *      packet.
+     * </p>
+     *
+     * <b>
+     *     TODO:
+     *     the current implementation, which uses a set of previously seen sequence numbers, will consider a segment
+     *     with a reused sequence number---occurring as a result of sequence number wrap around for a very long-lived
+     *     connection---as a retransmission (and may therefore end up discarding it even though it is in fact NOT a
+     *     retransmission). Ideas?
+     * </b>
+     *
+     * @param packet The packet.
+     * @return {@code true} if {@code packet} was determined to be a retransmission, {@code false} otherwise.
+     */
+    public boolean isRetransmission(PcapPacket packet) {
+        // Extract sequence number.
+        int seqNo = packet.get(TcpPacket.class).getHeader().getSequenceNumber();
+        switch (getDirection(packet)) {
+            case CLIENT_TO_SERVER:
+                return mSeqNumbersClient.contains(seqNo);
+            case SERVER_TO_CLIENT:
+                return mSeqNumbersSrv.contains(seqNo);
+            default:
+                throw new AssertionError(String.format("Unexpected value of enum '%s'",
+                        Direction.class.getSimpleName()));
+        }
+    }
+
+    /**
+     * <p>
+     *     Is this {@code Conversation} a TLS session?
+     * </p>
+     *
+     * <em>Note: the current implementation simply examines the port number(s) for 443; it does <b>not</b> verify if the
+     * application data is indeed encrypted.</em>
+     *
+     * @return {@code true} if this {@code Conversation} is interpreted as a TLS session, {@code false} otherwise.
+     */
+    public boolean isTls() {
+        /*
+         * TODO:
+         * - may want to change this to be "return mServerPort == 443 || mClientPort == 443;" in order to also detect
+         *   TLS in those cases where it is not possible to correctly label who is the client and who is the server,
+         *   i.e., when the trace does not contain the SYN/SYNACK exchange.
+         * - current implementation relies on the server using the conventional TLS port number; may instead want to
+         *   inspect the first 4 bytes of each potential TLS packet to see if they match the SSL record header.
+         *
+         * 08/31/18: Added unconvetional TLS ports used by WeMo plugs and LiFX bulb.
+         * 09/20/18: Moved hardcoded ports to other class to allow other classes to query the set of TLS ports.
+         */
+        return TcpConversationUtils.isTlsPort(mServerPort);
+    }
+
+    /**
+     * If this {@code Conversation} is backing a TLS session (i.e., if the value of {@link #isTls()} is {@code true}),
+     * get the packets labeled as TLS Application Data packets. This is a subset of the full set of payload-carrying
+     * packets (as returned by {@link #getPackets()}). An exception is thrown if this method is invoked on a
+     * {@code Conversation} for which {@link #isTls()} returns {@code false}.
+     *
+     * @return A list containing exactly those packets that could be identified as TLS Application Data packets (through
+     *         inspecting of the SSL record header). The list may be empty, if no TLS application data packets have been
+     *         recorded for this {@code Conversation}.
+     */
+    public List<PcapPacket> getTlsApplicationDataPackets() {
+        if (!isTls()) {
+            throw new NoSuchElementException("cannot get TLS Application Data packets for non-TLS TCP conversation");
+        }
+        return Collections.unmodifiableList(mTlsApplicationDataPackets);
+    }
+
+    /**
+     * Extracts the TCP sequence number from {@code packet} and adds it to the proper set of sequence numbers by
+     * analyzing the direction of the packet.
+     * @param packet A TCP packet (wrapped in a {@code PcapPacket}) that was added to this conversation and whose
+     *               sequence number is to be recorded as seen.
+     */
+    private void addSeqNumber(PcapPacket packet) {
+        // Note: below check is redundant if client code is correct as the call to check the precondition should already
+        // have been made by the addXPacket method that invokes this method. As such, the call below may be removed in
+        // favor of speed, but the improvement will be minor, hence the added safety may be worth it.
+        onAddPrecondition(packet);
+        // Extract sequence number.
+        int seqNo = packet.get(TcpPacket.class).getHeader().getSequenceNumber();
+        // Determine direction of packet and add packet's sequence number to corresponding set of sequence numbers.
+        switch (getDirection(packet)) {
+            case CLIENT_TO_SERVER:
+                // Client to server packet.
+                mSeqNumbersClient.add(seqNo);
+                break;
+            case SERVER_TO_CLIENT:
+                // Server to client packet.
+                mSeqNumbersSrv.add(seqNo);
+                break;
+            default:
+                throw new AssertionError(String.format("Unexpected value of enum '%s'",
+                        Direction.class.getSimpleName()));
+        }
+    }
+
+    /**
+     * Determine the direction of {@code packet}. An {@link IllegalArgumentException} is thrown if {@code packet} does
+     * not pertain to this conversation.
+     *
+     * @param packet The packet whose direction is to be determined.
+     * @return A {@link Direction} indicating the direction of the packet.
+     */
+    public Direction getDirection(PcapPacket packet) {
+        IpV4Packet ipPacket = packet.get(IpV4Packet.class);
+        String ipSrc = ipPacket.getHeader().getSrcAddr().getHostAddress();
+        String ipDst = ipPacket.getHeader().getDstAddr().getHostAddress();
+        // Determine direction of packet.
+        if (ipSrc.equals(mClientIp) && ipDst.equals(mServerIp)) {
+            // Client to server packet.
+            return Direction.CLIENT_TO_SERVER;
+        } else if (ipSrc.equals(mServerIp) && ipDst.equals(mClientIp)) {
+            // Server to client packet.
+            return Direction.SERVER_TO_CLIENT;
+        } else {
+            throw new IllegalArgumentException("getDirection: packet not related to " + getClass().getSimpleName());
+        }
+    }
+
+    /**
+     * Utility enum for expressing the direction of a packet pertaining to this {@code Conversation}.
+     */
+    public enum Direction {
+
+        CLIENT_TO_SERVER {
+            @Override
+            public String toCompactString() {
+                return "*";
+            }
+        },
+        SERVER_TO_CLIENT {
+            @Override
+            public String toCompactString() {
+                return "";
+            }
+        };
+
+        /**
+         * Get a compact string representation of this {@code Direction}.
+         * @return a compact string representation of this {@code Direction}.
+         */
+        abstract public String toCompactString();
+
+    }
+
+}
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/FinAckPair.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/FinAckPair.java
new file mode 100644 (file)
index 0000000..fe4f032
--- /dev/null
@@ -0,0 +1,134 @@
+package edu.uci.iotproject.trafficreassembly.layer3;
+
+import org.pcap4j.core.PcapPacket;
+import org.pcap4j.packet.IpV4Packet;
+import org.pcap4j.packet.TcpPacket;
+
+/**
+ * Groups a FIN packet and its corresponding ACK packet. <b>Immutable and thread safe</b>.
+ *
+ * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
+ * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
+ */
+public class FinAckPair {
+
+    private final PcapPacket mFinPacket;
+    private final PcapPacket mCorrespondingAckPacket;
+
+    /**
+     * Constructs a {@code FinAckPair} given a FIN packet.
+     * The corresponding ACK packet field is set to {@code null}.
+     * @param finPacket A FIN packet.
+     */
+    public FinAckPair(PcapPacket finPacket) {
+        if (!finPacket.get(TcpPacket.class).getHeader().getFin()) {
+            throw new IllegalArgumentException("not a FIN packet");
+        }
+        mFinPacket = finPacket;
+        mCorrespondingAckPacket = null;
+    }
+
+    /**
+     * Constructs a {@code FinAckPair} given a FIN and an ACK packet.
+     * @param finPacket A FIN packet.
+     * @param correspondingAckPacket The ACK packet corresponding to {@code finPacket}.
+     */
+    public FinAckPair(PcapPacket finPacket, PcapPacket correspondingAckPacket) {
+        // Enforce class invariant, i.e. that the FIN and ACK are related.
+        // Note that it is indirectly checked whether finPacket is indeed a FIN packet
+        // as isCorrespondingAckPacket calls the single parameter constructor.
+        if (!FinAckPair.isCorrespondingAckPacket(finPacket, correspondingAckPacket)) {
+            throw new IllegalArgumentException("FIN and ACK not related");
+        }
+        mFinPacket = finPacket;
+        mCorrespondingAckPacket = correspondingAckPacket;
+    }
+
+    /**
+     * Get the FIN packet of this pair.
+     * @return the FIN packet of this pair.
+     */
+    public PcapPacket getFinPacket() {
+        return mFinPacket;
+    }
+
+    /**
+     * Get the corresponding ACK packet of this pair, if any.
+     * @return the corresponding ACK packet of this pair, if any.
+     */
+    public PcapPacket getCorrespondingAckPacket() {
+        return mCorrespondingAckPacket;
+    }
+
+    /**
+     * Was the FIN in this {@code FinAckPair} acknowledged?
+     *
+     * @return {@code true} if the corresponding ACK has been set in this {@code FinAckPair}.
+     */
+    public boolean isAcknowledged() {
+        return mFinPacket != null && mCorrespondingAckPacket != null;
+    }
+
+    /**
+     * Checks if a given packet is an ACK corresponding to the FIN packet in this {@code FinAckPair}.
+     * @return {@code true} if {@code packet} is an ACK that corresponds to the FIN in this pair, {@code false} otherwise.
+     */
+    public boolean isCorrespondingAckPacket(PcapPacket packet) {
+        IpV4Packet inputIpPacket = packet.get(IpV4Packet.class);
+        TcpPacket inputTcpPacket = packet.get(TcpPacket.class);
+        if (inputIpPacket == null || inputTcpPacket == null || !inputTcpPacket.getHeader().getAck()) {
+            return false;
+        }
+
+        IpV4Packet finIpPacket = mFinPacket.get(IpV4Packet.class);
+        TcpPacket finTcpPacket = mFinPacket.get(TcpPacket.class);
+
+        // Extract (srcIp:port,dstIp:port) for input and member (FIN) packets.
+        String inputPacketIpSrc = inputIpPacket.getHeader().getSrcAddr().getHostAddress();
+        String inputPacketIpDst = inputIpPacket.getHeader().getDstAddr().getHostAddress();
+        int inputPacketPortSrc = inputTcpPacket.getHeader().getSrcPort().valueAsInt();
+        int inputPacketPortDst = inputTcpPacket.getHeader().getDstPort().valueAsInt();
+        String finPacketIpSrc = finIpPacket.getHeader().getSrcAddr().getHostAddress();
+        String finPacketIpDst = finIpPacket.getHeader().getDstAddr().getHostAddress();
+        int finPacketPortSrc = finTcpPacket.getHeader().getSrcPort().valueAsInt();
+        int finPacketPortDst = finTcpPacket.getHeader().getDstPort().valueAsInt();
+
+        // For the two packets to be related, the dst of one must be the src of the other.
+        // Split into multiple if statements for readability. First check IP fields, then ports.
+        if (!(inputPacketIpDst.equals(finPacketIpSrc) && finPacketIpDst.equals(inputPacketIpSrc))) {
+            return false;
+        }
+        if (!(inputPacketPortDst == finPacketPortSrc && finPacketPortDst == inputPacketPortSrc)) {
+            return false;
+        }
+
+        // Packets are (most likely) related (part of same conversation/stream).
+        // Now all that is left for us to check is if the sequence numbers match.
+        // Note: recall that the FIN packet advances the seq numbers by 1,
+        // so the ACK number will be one larger than the seq. number in the FIN packet.
+        return inputTcpPacket.getHeader().getAcknowledgmentNumber() == finTcpPacket.getHeader().getSequenceNumber() + 1;
+    }
+
+    /**
+     * Static method to check if two given packets are a FIN and the corresponding ACK packet.
+     * The purpose of this method is a workaround to enforce the class invariant in the two parameter constructor.
+     * Specifically, the following should be avoided:
+     * <pre>
+     *     public FinAckPair(PcapPacket finPacket, PcapPacket correspondingAckPacket) {
+     *         mFinPacket = finPacket;
+     *         // Below line is considered bad practice as the object has not been fully initialized at this stage.
+     *         if (!this.isCorrespondingAckPacket(correspondingAckPacket)) {
+     *             // ... throw exception
+     *         }
+     *     }
+     * </pre>
+     * @param finPacket The FIN packet.
+     * @param ackPacket The ACK packet that is to be checked if it corresponds to the given FIN packet.
+     * @return {@code true} if the ACK corresponds to the FIN, {@code false} otherwise.
+     */
+    private static boolean isCorrespondingAckPacket(PcapPacket finPacket, PcapPacket ackPacket) {
+        FinAckPair tmp = new FinAckPair(finPacket);
+        return tmp.isCorrespondingAckPacket(ackPacket);
+    }
+
+}
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/TcpReassembler.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/TcpReassembler.java
new file mode 100644 (file)
index 0000000..e150875
--- /dev/null
@@ -0,0 +1,260 @@
+package edu.uci.iotproject.trafficreassembly.layer3;
+
+import org.pcap4j.core.PacketListener;
+import org.pcap4j.core.PcapPacket;
+import org.pcap4j.packet.*;
+
+import java.util.*;
+
+/**
+ * Reassembles TCP conversations (streams).
+ * <b>Note: current version only supports TCP over IPv4.</b>
+ *
+ * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
+ * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
+ */
+public class TcpReassembler implements PacketListener {
+
+    /**
+     * Holds <em>open</em> {@link Conversation}s, i.e., {@code Conversation}s that have <em>not</em> been detected as
+     * (gracefully) terminated based on the set of packets observed thus far.
+     * A {@link Conversation} is moved to {@link #mTerminatedConversations} if it can be determined that it is has
+     * terminated. Termination can be detected by a) observing two {@link FinAckPair}s, one in each direction, (graceful
+     * termination, see {@link Conversation#isGracefullyShutdown()}) or b) by observing a SYN packet that matches the
+     * four tuple of an existing {@code Conversation}, but which holds a <em>different</em> sequence number than the
+     * same-direction SYN packet recorded for the {@code Conversation}.
+     * <p>
+     * Note that due to limitations of the {@link Set} interface (specifically, there is no {@code get(T t)} method),
+     * we have to resort to a {@link Map} (in which keys map to themselves) to "mimic" a set with {@code get(T t)}
+     * functionality.
+     *
+     * @see <a href="https://stackoverflow.com/questions/7283338/getting-an-element-from-a-set">this question on StackOverflow.com</a>
+     */
+    private final Map<Conversation, Conversation> mOpenConversations = new HashMap<>();
+
+    /**
+     * Holds <em>terminated</em> {@link Conversation}s.
+     */
+    private final List<Conversation> mTerminatedConversations = new ArrayList<>();
+
+    @Override
+    public void gotPacket(PcapPacket pcapPacket) {
+        IpV4Packet ipPacket = pcapPacket.get(IpV4Packet.class);
+        TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class);
+
+        if (ipPacket == null || tcpPacket == null) {
+            return;
+        }
+        // ... TODO?
+        processPacket(pcapPacket);
+//        Class clazz = pcapPacket.getClass();
+//        RadiotapPacket radiotapPacket = pcapPacket.get(RadiotapPacket.class);
+//        Dot11ManagementPacket dot11ManagementPacket = pcapPacket.get(Dot11ManagementPacket.class);
+//        if (dot11ManagementPacket != null) {
+//            return;
+//        }
+//        if (radiotapPacket != null) {
+//            processRadiotapPacket(pcapPacket);
+//        }
+    }
+
+    /**
+     * Get the reassembled TCP connections. Note that if this is called while packets are still being processed (by
+     * calls to {@link #gotPacket(PcapPacket)}), the behavior is undefined and the returned list may be inconsistent.
+     * @return The reassembled TCP connections.
+     */
+    public List<Conversation> getTcpConversations() {
+        ArrayList<Conversation> combined = new ArrayList<>();
+        combined.addAll(mTerminatedConversations);
+        combined.addAll(mOpenConversations.values());
+        return combined;
+    }
+
+    private void processRadiotapPacket(PcapPacket pcapPacket) {
+        RadiotapPacket radiotapPacket = pcapPacket.get(RadiotapPacket.class);
+
+        RadiotapPacket.RadiotapHeader header = radiotapPacket.getHeader();
+        short length = header.getLength();
+        ArrayList<RadiotapPacket.RadiotapData> radiotapData = header.getDataFields();
+        // TODO: We can handle this 802.11 QoS data by creating our own class
+        // TODO: We only need to handle the first few bytes for source, destination, receiver, and transmitter
+        // TODO: addresses
+        Packet dataPacket = radiotapPacket.getPayload();
+        int dataLength = dataPacket.length();
+    }
+
+    private void processPacket(PcapPacket pcapPacket) {
+        TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class);
+        // Handle client connection initiation attempts.
+        if (tcpPacket.getHeader().getSyn() && !tcpPacket.getHeader().getAck()) {
+            // A segment with the SYN flag set, but no ACK flag indicates that a client is attempting to initiate a new
+            // connection.
+            processNewConnectionRequest(pcapPacket);
+            return;
+        }
+        // Handle server connection initiation acknowledgement
+        if (tcpPacket.getHeader().getSyn() && tcpPacket.getHeader().getAck()) {
+            // A segment with both the SYN and ACK flags set indicates that the server has accepted the client's request
+            // to initiate a new connection.
+            processNewConnectionAck(pcapPacket);
+            return;
+        }
+        // Handle resets
+        if (tcpPacket.getHeader().getRst()) {
+            processRstPacket(pcapPacket);
+            return;
+        }
+        // Handle FINs
+        if (tcpPacket.getHeader().getFin()) {
+            // Handle FIN packet.
+            processFinPacket(pcapPacket);
+        }
+        // Handle ACKs (currently only ACKs of FINS)
+        if (tcpPacket.getHeader().getAck()) {
+            processAck(pcapPacket);
+        }
+        // Handle packets that carry payload (application data).
+        if (tcpPacket.getPayload() != null) {
+            processPayloadPacket(pcapPacket);
+        }
+    }
+
+    private void processNewConnectionRequest(PcapPacket clientSynPacket) {
+        // A SYN w/o ACK always originates from the client.
+        Conversation conv = Conversation.fromPcapPacket(clientSynPacket, true);
+        conv.addSynPacket(clientSynPacket);
+        // Is there an ongoing conversation for the same four tuple (clientIp, clientPort, serverIp, serverPort) as
+        // found in the new SYN packet?
+        Conversation ongoingConv = mOpenConversations.get(conv);
+        if (ongoingConv != null) {
+            if (ongoingConv.isRetransmission(clientSynPacket)) {
+                // SYN retransmission detected, do nothing.
+                return;
+                // TODO: the way retransmission detection is implemented may cause a bug for connections where we have
+                // not recorded the initial SYN, but only the SYN ACK, as retransmission is determined by comparing the
+                // sequence numbers of initial SYNs -- and if no initial SYN is present for the Conversation, the new
+                // SYN will be interpreted as a retransmission. Possible fix: let isRentransmission ALWAYS return false
+                // when presented with a SYN packet when the Conversation already holds a SYN ACK packet?
+            } else {
+                // New SYN has different sequence number than SYN recorded for ongoingConv, so this must be an attempt
+                // to establish a new conversation with the same four tuple as ongoingConv.
+                // Mark existing connection as terminated.
+                // TODO: is this 100% theoretically correct, e.g., if many connection attempts are made back to back? And RST packets?
+                mTerminatedConversations.add(ongoingConv);
+                mOpenConversations.remove(ongoingConv);
+            }
+        }
+        // Finally, update the map of open connections with the new connection.
+        mOpenConversations.put(conv, conv);
+    }
+
+
+    /*
+     * TODO a problem across the board for all processXPacket methods below:
+     * if we start the capture in the middle of a TCP connection, we will not have an entry for the conversation in the
+     * map as we have not seen the initial SYN packet.
+     * Two ways we can address this:
+     * a) Perform null-checks and ignore packets for which we have not seen SYN
+     *    + easy to get correct
+     *    - we discard data (issue for long-lived connections!)
+     * b) Add a corresponding conversation entry whenever we encounter a packet that does not map to a conversation
+     *    + we consider all data
+     *    - not immediately clear if this will introduce bugs (incorrectly mapping packets to wrong conversations?)
+     *
+     *  [[[ I went with option b) for now; see getOngoingConversationOrCreateNew(PcapPacket pcapPacket). ]]]
+     */
+
+    private void processNewConnectionAck(PcapPacket srvSynPacket) {
+        // Find the corresponding ongoing connection, if any (if we start the capture just *after* the initial SYN, no
+        // ongoing conversation entry will exist, so it must be created in that case).
+//        Conversation conv = mOpenConversations.get(Conversation.fromPcapPacket(srvSynPacket, false));
+        Conversation conv = getOngoingConversationOrCreateNew(srvSynPacket);
+        // Note: exploits &&'s short-circuit operation: only attempts to add non-retransmissions.
+        if (!conv.isRetransmission(srvSynPacket) && !conv.addSynPacket(srvSynPacket)) {
+            // For safety/debugging: if NOT a retransmission and add fails,
+            // something has gone terribly wrong/invariant is broken.
+//            throw new AssertionError("Attempt to add SYN ACK packet that was NOT a retransmission failed." +
+//                    Conversation.class.getSimpleName() + " invariant broken.");
+        }
+    }
+
+    private void processRstPacket(PcapPacket rstPacket) {
+        Conversation conv = getOngoingConversationOrCreateNew(rstPacket);
+        // Add RST packet to conversation.
+        conv.addRstPacket(rstPacket);
+        // Move conversation to set of terminated conversations.
+        mTerminatedConversations.add(conv);
+        mOpenConversations.remove(conv, conv);
+    }
+
+    private void processFinPacket(PcapPacket finPacket) {
+//        getOngoingConversationForPacket(finPacket).addFinPacket(finPacket);
+        getOngoingConversationOrCreateNew(finPacket).addFinPacket(finPacket);
+    }
+
+    private void processAck(PcapPacket ackPacket) {
+//        getOngoingConversationForPacket(ackPacket).attemptAcknowledgementOfFin(ackPacket);
+        // Note that unlike the style for SYN, FIN, and payload packets, for "ACK only" packets, we want to avoid
+        // creating a new conversation.
+        Conversation conv = getOngoingConversationForPacket(ackPacket);
+        if (conv != null) {
+            // The ACK may be an ACK of a FIN, so attempt to mark the FIN as ack'ed.
+            conv.attemptAcknowledgementOfFin(ackPacket);
+            if (conv.isGracefullyShutdown()) {
+                // Move conversation to set of terminated conversations.
+                mTerminatedConversations.add(conv);
+                mOpenConversations.remove(conv);
+            }
+        }
+        // Note: add (additional) processing of ACKs (that are not ACKs of FINs) as necessary here...
+    }
+
+    private void processPayloadPacket(PcapPacket pcapPacket) {
+//        getOngoingConversationForPacket(pcapPacket).addPacket(pcapPacket, true);
+        getOngoingConversationOrCreateNew(pcapPacket).addPacket(pcapPacket, true);
+    }
+
+    /**
+     * Locates an ongoing conversation (if any) that {@code pcapPacket} pertains to.
+     * @param pcapPacket The packet that is to be mapped to an ongoing {@code Conversation}.
+     * @return The {@code Conversation} matching {@code pcapPacket} or {@code null} if there is no match.
+     */
+    private Conversation getOngoingConversationForPacket(PcapPacket pcapPacket) {
+        // We cannot know if this is a client-to-server or server-to-client packet without trying both options...
+        Conversation conv = mOpenConversations.get(Conversation.fromPcapPacket(pcapPacket, true));
+        if (conv == null) {
+            conv = mOpenConversations.get(Conversation.fromPcapPacket(pcapPacket, false));
+        }
+        return conv;
+    }
+
+    /**
+     * Like {@link #getOngoingConversationForPacket(PcapPacket)}, but creates and inserts a new {@code Conversation}
+     * into {@link #mOpenConversations} if no open conversation is found (i.e., in the case that
+     * {@link #getOngoingConversationForPacket(PcapPacket)} returns {@code null}).
+     *
+     * @param pcapPacket The packet that is to be mapped to an ongoing {@code Conversation}.
+     * @return The existing, ongoing {@code Conversation} matching {@code pcapPacket} or the newly created one in case
+     *         no match was found.
+     */
+    private Conversation getOngoingConversationOrCreateNew(PcapPacket pcapPacket) {
+        Conversation conv = getOngoingConversationForPacket(pcapPacket);
+        if (conv == null) {
+            TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class);
+            if (tcpPacket.getHeader().getSyn() && tcpPacket.getHeader().getAck()) {
+                // A SYN ACK packet always originates from the server (it is a reply to the initial SYN packet from the client)
+                conv = Conversation.fromPcapPacket(pcapPacket, false);
+            } else {
+                // TODO: can we do anything else but arbitrarily select who is designated as the server in this case?
+                // We can check if the IP prefix matches a local IP when handling traffic observed inside the local
+                // network, but that obviously won't be a useful strategy for an observer at the WAN port.
+                String srcIp = pcapPacket.get(IpV4Packet.class).getHeader().getSrcAddr().getHostAddress();
+                // TODO: REPLACE THE ROUTER'S IP WITH A PARAMETER!!!
+                boolean clientIsSrc = srcIp.startsWith("10.") || srcIp.startsWith("192.168.") || srcIp.equals("128.195.205.105");
+                conv = Conversation.fromPcapPacket(pcapPacket, clientIsSrc);
+            }
+            mOpenConversations.put(conv, conv);
+        }
+        return conv;
+    }
+}
index 8f58d977850036d9c71bd18df05225412c3636ca..cee09fed8c1c78ddc19459470864b01571f6fd93 100644 (file)
@@ -1,13 +1,15 @@
 package edu.uci.iotproject.util;
 
-import edu.uci.iotproject.Conversation;
+import edu.uci.iotproject.trafficreassembly.layer3.Conversation;
 import edu.uci.iotproject.analysis.PcapPacketPair;
 import edu.uci.iotproject.analysis.TcpConversationUtils;
 import edu.uci.iotproject.analysis.TriggerTrafficExtractor;
 import org.apache.commons.math3.stat.clustering.Cluster;
 import org.pcap4j.core.PcapPacket;
+import org.pcap4j.packet.EthernetPacket;
 import org.pcap4j.packet.IpV4Packet;
 import org.pcap4j.packet.TcpPacket;
+import org.pcap4j.util.MacAddress;
 
 import java.util.*;
 
@@ -26,6 +28,25 @@ public final class PcapPacketUtils {
      */
     private static final int SIGNATURE_MERGE_THRESHOLD = 5;
 
+
+    /**
+     * Gets the source address of the Ethernet part of {@code packet}.
+     * @param packet The packet for which the Ethernet source address is to be extracted.
+     * @return The source address of the Ethernet part of {@code packet}.
+     */
+    public static MacAddress getEthSrcAddr(PcapPacket packet) {
+        return getEthernetPacketOrThrow(packet).getHeader().getSrcAddr();
+    }
+
+    /**
+     * Gets the destination address of the Ethernet part of {@code packet}.
+     * @param packet The packet for which the Ethernet destination address is to be extracted.
+     * @return The destination address of the Ethernet part of {@code packet}.
+     */
+    public static MacAddress getEthDstAddr(PcapPacket packet) {
+        return getEthernetPacketOrThrow(packet).getHeader().getDstAddr();
+    }
+
     /**
      * Determines if a given {@link PcapPacket} wraps a {@link TcpPacket}.
      * @param packet The {@link PcapPacket} to inspect.
@@ -346,7 +367,7 @@ public final class PcapPacketUtils {
     /**
      * Gets the {@link IpV4Packet} contained in {@code packet}, or throws a {@link NullPointerException} if
      * {@code packet} does not contain an {@link IpV4Packet}.
-     * @param packet A {@link PcapPacket} that is expected to contain a {@link IpV4Packet}.
+     * @param packet A {@link PcapPacket} that is expected to contain an {@link IpV4Packet}.
      * @return The {@link IpV4Packet} contained in {@code packet}.
      * @throws NullPointerException if {@code packet} does not encapsulate an {@link IpV4Packet}.
      */
@@ -354,6 +375,17 @@ public final class PcapPacketUtils {
         return Objects.requireNonNull(packet.get(IpV4Packet.class), "not an IPv4 packet");
     }
 
+    /**
+     * Gets the {@link EthernetPacket} contained in {@code packet}, or throws a {@link NullPointerException} if
+     * {@code packet} does not contain an {@link EthernetPacket}.
+     * @param packet A {@link PcapPacket} that is expected to contain an {@link EthernetPacket}.
+     * @return The {@link EthernetPacket} contained in {@code packet}.
+     * @throws NullPointerException if {@code packet} does not encapsulate an {@link EthernetPacket}.
+     */
+    private static final EthernetPacket getEthernetPacketOrThrow(PcapPacket packet) {
+        return Objects.requireNonNull(packet.get(EthernetPacket.class), "not an Ethernet packet");
+    }
+
     /**
      * Print signatures in {@code List} of {@code List} of {@code List} of {@code PcapPacket} objects.
      *