Reorganize code by creating a package for code that reassembles traffic flows at...
authorJanus Varmarken <varmarken@gmail.com>
Sun, 13 Jan 2019 04:18:53 +0000 (20:18 -0800)
committerJanus Varmarken <varmarken@gmail.com>
Sun, 13 Jan 2019 04:18:53 +0000 (20:18 -0800)
31 files changed:
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Conversation.java [deleted file]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FinAckPair.java [deleted file]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FlowPatternFinder.java
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/L2FlowReassembler.java [deleted file]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Layer2Flow.java [deleted file]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Layer2FlowReassemblerObserver.java [deleted file]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/TcpReassembler.java [deleted file]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TcpConversationUtils.java
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TrafficLabeler.java
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/AbstractPatternComparisonResult.java
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/ComparisonFunctions.java
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/CompleteMatchPatternComparisonResult.java
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/PatternComparisonTask.java
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/seqalignment/ExtractedSequence.java
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/seqalignment/SequenceExtraction.java
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/AbstractClusterMatcher.java
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/ClusterMatcher.java
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/Layer2ClusterMatcher.java [deleted file]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/Layer2FlowObserver.java
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/Layer2SequenceMatcher.java [deleted file]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer2/Layer2ClusterMatcher.java [new file with mode: 0644]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SequenceMatcher.java [new file with mode: 0644]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/evaluation/SanitySignatureGenerator.java
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2Flow.java [new file with mode: 0644]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowReassembler.java [new file with mode: 0644]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowReassemblerObserver.java [new file with mode: 0644]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/Conversation.java [new file with mode: 0644]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/FinAckPair.java [new file with mode: 0644]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/TcpReassembler.java [new file with mode: 0644]
Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java

diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Conversation.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Conversation.java
deleted file mode 100644 (file)
index 490dc4f..0000000
+++ /dev/null
@@ -1,585 +0,0 @@
-package edu.uci.iotproject;
-
-import edu.uci.iotproject.analysis.TcpConversationUtils;
-import edu.uci.iotproject.util.PcapPacketUtils;
-import org.pcap4j.core.PcapPacket;
-import org.pcap4j.packet.IpV4Packet;
-import org.pcap4j.packet.Packet;
-import org.pcap4j.packet.TcpPacket;
-
-import java.util.*;
-
-/**
- * Models a (TCP) conversation/connection/session/flow (packet's belonging to the same session between a client and a
- * server).
- * Holds a list of {@link PcapPacket}s identified as pertaining to the flow. Note that this list is <em>not</em>
- * considered when determining equality of two {@code Conversation} instances in order to allow for a
- * {@code Conversation} to function as a key in data structures such as {@link java.util.Map} and {@link java.util.Set}.
- * See {@link #equals(Object)} for the definition of equality.
- *
- * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
- * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
- */
-public class Conversation {
-
-    /* Begin instance properties */
-    /**
-     * The IP of the host that is considered the client (i.e. the host that initiates the conversation)
-     * in this conversation.
-     */
-    private final String mClientIp;
-
-    /**
-     * The port number used by the host that is considered the client in this conversation.
-     */
-    private final int mClientPort;
-
-    /**
-     * The IP of the host that is considered the server (i.e. is the responder) in this conversation.
-     */
-    private final String mServerIp;
-
-    /**
-     * The port number used by the server in this conversation.
-     */
-    private final int mServerPort;
-
-    /**
-     * The list of packets (with payload) pertaining to this conversation.
-     */
-    private final List<PcapPacket> mPackets;
-
-    /**
-     * If {@link #isTls()} is {@code true}, this list contains the subset of {@link #mPackets} which are TLS Application
-     * Data packets.
-     */
-    private final List<PcapPacket> mTlsApplicationDataPackets;
-
-    /**
-     * Contains the sequence numbers used thus far by the host that is considered the <em>client</em> in this
-     * {@code Conversation}.
-     * Used for filtering out retransmissions.
-     */
-    private final Set<Integer> mSeqNumbersClient;
-
-    /**
-     * Contains the sequence numbers used thus far by the host that is considered the <em>server</em> in this
-     * {@code Conversation}.
-     * Used for filtering out retransmissions.
-     */
-    private final Set<Integer> mSeqNumbersSrv;
-
-    /**
-     * List of SYN packets pertaining to this conversation.
-     */
-    private final List<PcapPacket> mSynPackets;
-
-    /**
-     * List of pairs FINs and their corresponding ACKs associated with this conversation.
-     */
-    private final List<FinAckPair> mFinPackets;
-
-    /**
-     * List of RST packets associated with this conversation.
-     */
-    private final List<PcapPacket> mRstPackets;
-
-    /**
-     * Boolean to mark the packet as Application Data based on the previous packet that reaches MTU
-     */
-    private boolean mApplicationData;
-    /* End instance properties */
-
-    /**
-     * Factory method for creating a {@code Conversation} from a {@link PcapPacket}.
-     * @param pcapPacket The {@code PcapPacket} that wraps a TCP segment for which a {@code Conversation} is to be initiated.
-     * @param clientIsSrc If {@code true}, the source address and source port found in the IP datagram and TCP segment
-     *                    wrapped in the {@code PcapPacket} are regarded as pertaining to the client, and the destination
-     *                    address and destination port are regarded as pertaining to the server---and vice versa if set
-     *                    to {@code false}.
-     * @return A {@code Conversation} initiated with ip:port for client and server according to the direction of the packet.
-     */
-    public static Conversation fromPcapPacket(PcapPacket pcapPacket, boolean clientIsSrc) {
-        IpV4Packet ipPacket = pcapPacket.get(IpV4Packet.class);
-        TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class);
-        String clientIp = clientIsSrc ? ipPacket.getHeader().getSrcAddr().getHostAddress() :
-                ipPacket.getHeader().getDstAddr().getHostAddress();
-        String srvIp = clientIsSrc ? ipPacket.getHeader().getDstAddr().getHostAddress() :
-                ipPacket.getHeader().getSrcAddr().getHostAddress();
-        int clientPort = clientIsSrc ? tcpPacket.getHeader().getSrcPort().valueAsInt() :
-                tcpPacket.getHeader().getDstPort().valueAsInt();
-        int srvPort = clientIsSrc ? tcpPacket.getHeader().getDstPort().valueAsInt() :
-                tcpPacket.getHeader().getSrcPort().valueAsInt();
-        return new Conversation(clientIp, clientPort, srvIp, srvPort);
-    }
-
-    /**
-     * Constructs a new {@code Conversation}.
-     * @param clientIp The IP of the host that is considered the client (i.e. the host that initiates the conversation)
-     *                 in the conversation.
-     * @param clientPort The port number used by the client for the conversation.
-     * @param serverIp The IP of the host that is considered the server (i.e. is the responder) in the conversation.
-     * @param serverPort The port number used by the server for the conversation.
-     */
-    public Conversation(String clientIp, int clientPort, String serverIp, int serverPort) {
-        this.mClientIp = clientIp;
-        this.mClientPort = clientPort;
-        this.mServerIp = serverIp;
-        this.mServerPort = serverPort;
-        this.mPackets = new ArrayList<>();
-        this.mTlsApplicationDataPackets = new ArrayList<>();
-        this.mSeqNumbersClient = new HashSet<>();
-        this.mSeqNumbersSrv = new HashSet<>();
-        this.mSynPackets = new ArrayList<>();
-        this.mFinPackets = new ArrayList<>();
-        this.mRstPackets = new ArrayList<>();
-        this.mApplicationData = false;
-    }
-
-    /**
-     * Add a packet to the list of packets associated with this conversation.
-     * @param packet The packet that is to be added to (associated with) this conversation.
-     * @param ignoreRetransmissions Boolean value indicating if retransmissions should be ignored.
-     *                              If set to {@code true}, {@code packet} will <em>not</em> be added to the
-     *                              internal list of packets pertaining to this {@code Conversation}
-     *                              <em>iff</em> the sequence number of {@code packet} was already
-     *                              seen in a previous packet.
-     */
-    public void addPacket(PcapPacket packet, boolean ignoreRetransmissions) {
-        // Precondition: verify that packet does indeed pertain to conversation.
-        onAddPrecondition(packet);
-        if (ignoreRetransmissions && isRetransmission(packet)) {
-            // Packet is a retransmission. Ignore it.
-            return;
-        }
-        // Select direction-dependent set of sequence numbers seen so far and update it with sequence number of new packet.
-        addSeqNumber(packet);
-        // Finally add packet to list of packets pertaining to this conversation.
-        mPackets.add(packet);
-        // Preserve order of packets in list: sort according to timestamp.
-        if (mPackets.size() > 1 &&
-                mPackets.get(mPackets.size()-1).getTimestamp().isBefore(mPackets.get(mPackets.size()-2).getTimestamp())) {
-            Collections.sort(mPackets, (o1, o2) -> {
-                if (o1.getTimestamp().isBefore(o2.getTimestamp())) { return -1; }
-                else if (o2.getTimestamp().isBefore(o1.getTimestamp())) { return 1; }
-                else { return 0; }
-            });
-        }
-        // If TLS, inspect packet to see if it's a TLS Application Data packet, and if so add it to the list of TLS
-        // Application Data packets.
-        if (isTls()) {
-            TcpPacket tcpPacket = packet.get(TcpPacket.class);
-            Packet tcpPayload = tcpPacket.getPayload();
-            if (tcpPayload == null) {
-                return;
-            }
-            byte[] rawPayload = tcpPayload.getRawData();
-            // The SSL record header is at the front of the payload and is 5 bytes long.
-            // The SSL record header type field (the first byte) is set to 23 if it is an Application Data packet.
-            if (rawPayload != null && rawPayload.length >= 5) {
-                if (rawPayload[0] == 23) {
-                    mTlsApplicationDataPackets.add(packet);
-                    // Consider the following packet a data packet if this packet's size == MTU size 1448
-                    if (rawPayload.length >= 1448)
-                        mApplicationData = true;
-                } else if (rawPayload[0] == 20) {
-                    // Do nothing for now - CHANGE_CIPHER_SPEC
-                } else if (rawPayload[0] == 21) {
-                    // Do nothing for now - ALERT
-                } else if (rawPayload[0] == 22) {
-                    // Do nothing for now - HANDSHAKE
-                } else {
-                    // If it is TLS with payload, but rawPayload[0] != 23
-                    if (mApplicationData == true) {
-                        // It is a continuation of the previous packet if the previous packet reaches MTU size 1448 and
-                        // it is not either type 20, 21, or 22
-                        mTlsApplicationDataPackets.add(packet);
-                        if (rawPayload.length < 1448)
-                            mApplicationData = false;
-                    }
-                }
-            }
-        }
-    }
-
-    /**
-     * Get a list of packets pertaining to this {@code Conversation}.
-     * The returned list is a read-only list.
-     * @return the list of packets pertaining to this {@code Conversation}.
-     */
-    public List<PcapPacket> getPackets() {
-        // Return read-only view to prevent external code from manipulating internal state (preserve invariant).
-        return Collections.unmodifiableList(mPackets);
-    }
-
-    /**
-     * Records a TCP SYN packet as pertaining to this conversation (adds it to the the internal list).
-     * Attempts to add duplicate SYN packets will be ignored, and the caller is made aware of the attempt to add a
-     * duplicate by the return value being {@code false}.
-     *
-     * @param synPacket A {@link PcapPacket} wrapping a TCP SYN packet.
-     * @return {@code true} if the packet was successfully added to this {@code Conversation}, {@code false} otherwise.
-     */
-    public boolean addSynPacket(PcapPacket synPacket) {
-        onAddPrecondition(synPacket);
-        final IpV4Packet synPacketIpSection = synPacket.get(IpV4Packet.class);
-        final TcpPacket synPacketTcpSection = synPacket.get(TcpPacket.class);
-        if (synPacketTcpSection == null || !synPacketTcpSection.getHeader().getSyn()) {
-            throw new IllegalArgumentException("Not a SYN packet.");
-        }
-        // We are only interested in recording one copy of the two SYN packets (one SYN packet in each direction), i.e.,
-        // we want to discard retransmitted SYN packets.
-        if (mSynPackets.size() >= 2) {
-            return false;
-        }
-        // Check the set of recorded SYN packets to see if we have already recorded a SYN packet going in the same
-        // direction as the packet given in the argument.
-        boolean matchingPrevSyn = mSynPackets.stream().anyMatch(p -> {
-            IpV4Packet pIp = p.get(IpV4Packet.class);
-            TcpPacket pTcp = p.get(TcpPacket.class);
-            boolean srcAddrMatch = synPacketIpSection.getHeader().getSrcAddr().getHostAddress().
-                    equals(pIp.getHeader().getSrcAddr().getHostAddress());
-            boolean dstAddrMatch = synPacketIpSection.getHeader().getDstAddr().getHostAddress().
-                    equals(pIp.getHeader().getDstAddr().getHostAddress());
-            boolean srcPortMatch = synPacketTcpSection.getHeader().getSrcPort().valueAsInt() ==
-                    pTcp.getHeader().getSrcPort().valueAsInt();
-            boolean dstPortMatch = synPacketTcpSection.getHeader().getDstPort().valueAsInt() ==
-                    pTcp.getHeader().getDstPort().valueAsInt();
-            return srcAddrMatch && dstAddrMatch && srcPortMatch && dstPortMatch;
-        });
-        if (matchingPrevSyn) {
-            return false;
-        }
-        // Update direction-dependent set of sequence numbers and record/log packet.
-        addSeqNumber(synPacket);
-        return mSynPackets.add(synPacket);
-
-        /*
-        mSynPackets.stream().anyMatch(p -> {
-            IpV4Packet pIp = p.get(IpV4Packet.class);
-            TcpPacket pTcp = p.get(TcpPacket.class);
-            boolean srcAddrMatch = synPacketIpSection.getHeader().getSrcAddr().getHostAddress().
-                    equals(pIp.getHeader().getSrcAddr().getHostAddress());
-            boolean dstAddrMatch = synPacketIpSection.getHeader().getDstAddr().getHostAddress().
-                    equals(pIp.getHeader().getDstAddr().getHostAddress());
-            boolean srcPortMatch = synPacketTcpSection.getHeader().getSrcPort().valueAsInt() ==
-                    pTcp.getHeader().getSrcPort().valueAsInt();
-            boolean dstPortMatch = synPacketTcpSection.getHeader().getDstPort().value() ==
-                    pTcp.getHeader().getDstPort().value();
-
-            boolean fourTupleMatch = srcAddrMatch && dstAddrMatch && srcPortMatch && dstPortMatch;
-
-            boolean seqNoMatch = synPacketTcpSection.getHeader().getSequenceNumber() ==
-                    pTcp.getHeader().getSequenceNumber();
-
-            if (fourTupleMatch && !seqNoMatch) {
-                // If the four tuple that identifies the conversation matches, but the sequence number is different,
-                // it means that this SYN packet is, in fact, an attempt to establish a **new** connection, and hence
-                // the given packet is NOT part of this conversation, even though the ip:port combinations are (by
-                // chance) selected such that they match this conversation.
-                throw new IllegalArgumentException("Attempt to add SYN packet that belongs to a different conversation " +
-                        "(which is identified by the same four tuple as this conversation)");
-            }
-            return fourTupleMatch && seqNoMatch;
-        });
-        */
-    }
-
-    /**
-     * Get a list of SYN packets pertaining to this {@code Conversation}.
-     * The returned list is a read-only list.
-     * @return the list of SYN packets pertaining to this {@code Conversation}.
-     */
-    public List<PcapPacket> getSynPackets() {
-        return Collections.unmodifiableList(mSynPackets);
-    }
-
-    /**
-     * Adds a TCP FIN packet to the list of TCP FIN packets associated with this conversation.
-     * @param finPacket The TCP FIN packet that is to be added to (associated with) this conversation.
-     */
-    public void addFinPacket(PcapPacket finPacket) {
-        // Precondition: verify that packet does indeed pertain to conversation.
-        onAddPrecondition(finPacket);
-        // TODO: should call addSeqNumber here?
-        addSeqNumber(finPacket);
-        mFinPackets.add(new FinAckPair(finPacket));
-    }
-
-    /**
-     * Attempt to ACK any FIN packets held by this conversation.
-     * @param ackPacket The ACK for a FIN previously added to this conversation.
-     */
-    public void attemptAcknowledgementOfFin(PcapPacket ackPacket) {
-        // Precondition: verify that the packet pertains to this conversation.
-        onAddPrecondition(ackPacket);
-        // Mark unack'ed FIN(s) that this ACK matches as ACK'ed (there might be more than one in case of retransmissions..?)
-        mFinPackets.replaceAll(finAckPair -> !finAckPair.isAcknowledged() && finAckPair.isCorrespondingAckPacket(ackPacket) ? new FinAckPair(finAckPair.getFinPacket(), ackPacket) : finAckPair);
-    }
-
-    /**
-     * Retrieves an unmodifiable view of the list of {@link FinAckPair}s associated with this {@code Conversation}.
-     * @return an unmodifiable view of the list of {@link FinAckPair}s associated with this {@code Conversation}.
-     */
-    public List<FinAckPair> getFinAckPairs() {
-        return Collections.unmodifiableList(mFinPackets);
-    }
-
-    /**
-     * Get if this {@code Conversation} is considered to have been gracefully shut down.
-     * A {@code Conversation} has been gracefully shut down if it contains a FIN+ACK pair for both directions
-     * (client to server, and server to client).
-     * @return {@code true} if the connection has been gracefully shut down, false otherwise.
-     */
-    public boolean isGracefullyShutdown() {
-        //  The conversation has been gracefully shut down if we have recorded a FIN from both the client and the server which have both been ack'ed.
-        return mFinPackets.stream().anyMatch(finAckPair -> finAckPair.isAcknowledged() && PcapPacketUtils.isSource(finAckPair.getFinPacket(), mClientIp, mClientPort)) &&
-                mFinPackets.stream().anyMatch(finAckPair -> finAckPair.isAcknowledged() && PcapPacketUtils.isSource(finAckPair.getFinPacket(), mServerIp, mServerPort));
-    }
-
-    /**
-     * Add a TCP segment for which the RST flag is set to this {@code Conversation}.
-     * @param packet A {@link PcapPacket} wrapping a TCP segment pertaining to this {@code Conversation} for which the
-     *               RST flag is set.
-     */
-    public void addRstPacket(PcapPacket packet) {
-        /*
-         * TODO:
-         * When now also keeping track of RST packets, should we also...?
-         * 1) Prevent later packets from being added once a RST segment has been added?
-         * 2) Extend 'isGracefullyShutdown()' to also consider RST segments, or add another method, 'isShutdown()' that
-         *    both considers FIN/ACK (graceful) as well as RST (abrupt/"ungraceful") shutdown?
-         * 3) Should it be impossible to associate more than one RST segment with each Conversation?
-         */
-        onAddPrecondition(packet);
-        TcpPacket tcpPacket = packet.get(TcpPacket.class);
-        if (tcpPacket == null || !tcpPacket.getHeader().getRst()) {
-            throw new IllegalArgumentException("not a RST packet");
-        }
-        mRstPackets.add(packet);
-    }
-
-    /**
-     * Get the TCP segments pertaining to this {@code Conversation} for which it was detected that the RST flag is set.
-     * @return the TCP segments pertaining to this {@code Conversation} for which it was detected that the RST flag is
-     *         set.
-     */
-    public List<PcapPacket> getRstPackets() {
-        return Collections.unmodifiableList(mRstPackets);
-    }
-
-    // =========================================================================================================
-    // We simply reuse equals and hashCode methods of String.class to be able to use this class as a key
-    // in a Map.
-
-    /**
-     * <em>Note:</em> currently, equality is determined based on pairwise equality of the elements of the four tuple
-     * ({@link #mClientIp}, {@link #mClientPort}, {@link #mServerIp}, {@link #mServerPort}) for {@code this} and
-     * {@code obj}.
-     * @param obj The object to test for equality with {@code this}.
-     * @return {@code true} if {@code obj} is considered equal to {@code this} based on the definition of equality given above.
-     */
-    @Override
-    public boolean equals(Object obj) {
-        return obj instanceof Conversation && this.toString().equals(obj.toString());
-    }
-
-    @Override
-    public int hashCode() {
-        return toString().hashCode();
-    }
-    // =========================================================================================================
-
-    @Override
-    public String toString() {
-        return String.format("%s:%d %s:%d", mClientIp, mClientPort, mServerIp, mServerPort);
-    }
-
-    /**
-     * Invoke to verify that the precondition holds when a caller attempts to add a packet to this {@code Conversation}.
-     * An {@link IllegalArgumentException} is thrown if the precondition is violated.
-     * @param packet the packet to be added to this {@code Conversation}
-     */
-    private void onAddPrecondition(PcapPacket packet) {
-        // Apply precondition to preserve class invariant: all packets in mPackets must match the 4 tuple that
-        // defines the conversation.
-        IpV4Packet ipPacket = Objects.requireNonNull(packet.get(IpV4Packet.class));
-        // For now we only support TCP flows.
-        TcpPacket tcpPacket = Objects.requireNonNull(packet.get(TcpPacket.class));
-        String ipSrc = ipPacket.getHeader().getSrcAddr().getHostAddress();
-        String ipDst = ipPacket.getHeader().getDstAddr().getHostAddress();
-        int srcPort = tcpPacket.getHeader().getSrcPort().valueAsInt();
-        int dstPort = tcpPacket.getHeader().getDstPort().valueAsInt();
-        String clientIp, serverIp;
-        int clientPort, serverPort;
-        if (ipSrc.equals(mClientIp)) {
-            clientIp = ipSrc;
-            clientPort = srcPort;
-            serverIp = ipDst;
-            serverPort = dstPort;
-        } else {
-            clientIp = ipDst;
-            clientPort = dstPort;
-            serverIp = ipSrc;
-            serverPort = srcPort;
-        }
-        if (!(clientIp.equals(mClientIp) && clientPort == mClientPort &&
-                serverIp.equals(mServerIp) && serverPort == mServerPort)) {
-            throw new IllegalArgumentException(
-                    String.format("Attempt to add packet that does not pertain to %s",
-                            Conversation.class.getSimpleName()));
-        }
-    }
-
-    /**
-     * <p>
-     *      Determines if the TCP packet contained in {@code packet} is a retransmission of a previously seen (logged)
-     *      packet.
-     * </p>
-     *
-     * <b>
-     *     TODO:
-     *     the current implementation, which uses a set of previously seen sequence numbers, will consider a segment
-     *     with a reused sequence number---occurring as a result of sequence number wrap around for a very long-lived
-     *     connection---as a retransmission (and may therefore end up discarding it even though it is in fact NOT a
-     *     retransmission). Ideas?
-     * </b>
-     *
-     * @param packet The packet.
-     * @return {@code true} if {@code packet} was determined to be a retransmission, {@code false} otherwise.
-     */
-    public boolean isRetransmission(PcapPacket packet) {
-        // Extract sequence number.
-        int seqNo = packet.get(TcpPacket.class).getHeader().getSequenceNumber();
-        switch (getDirection(packet)) {
-            case CLIENT_TO_SERVER:
-                return mSeqNumbersClient.contains(seqNo);
-            case SERVER_TO_CLIENT:
-                return mSeqNumbersSrv.contains(seqNo);
-            default:
-                throw new AssertionError(String.format("Unexpected value of enum '%s'",
-                        Direction.class.getSimpleName()));
-        }
-    }
-
-    /**
-     * <p>
-     *     Is this {@code Conversation} a TLS session?
-     * </p>
-     *
-     * <em>Note: the current implementation simply examines the port number(s) for 443; it does <b>not</b> verify if the
-     * application data is indeed encrypted.</em>
-     *
-     * @return {@code true} if this {@code Conversation} is interpreted as a TLS session, {@code false} otherwise.
-     */
-    public boolean isTls() {
-        /*
-         * TODO:
-         * - may want to change this to be "return mServerPort == 443 || mClientPort == 443;" in order to also detect
-         *   TLS in those cases where it is not possible to correctly label who is the client and who is the server,
-         *   i.e., when the trace does not contain the SYN/SYNACK exchange.
-         * - current implementation relies on the server using the conventional TLS port number; may instead want to
-         *   inspect the first 4 bytes of each potential TLS packet to see if they match the SSL record header.
-         *
-         * 08/31/18: Added unconvetional TLS ports used by WeMo plugs and LiFX bulb.
-         * 09/20/18: Moved hardcoded ports to other class to allow other classes to query the set of TLS ports.
-         */
-        return TcpConversationUtils.isTlsPort(mServerPort);
-    }
-
-    /**
-     * If this {@code Conversation} is backing a TLS session (i.e., if the value of {@link #isTls()} is {@code true}),
-     * get the packets labeled as TLS Application Data packets. This is a subset of the full set of payload-carrying
-     * packets (as returned by {@link #getPackets()}). An exception is thrown if this method is invoked on a
-     * {@code Conversation} for which {@link #isTls()} returns {@code false}.
-     *
-     * @return A list containing exactly those packets that could be identified as TLS Application Data packets (through
-     *         inspecting of the SSL record header). The list may be empty, if no TLS application data packets have been
-     *         recorded for this {@code Conversation}.
-     */
-    public List<PcapPacket> getTlsApplicationDataPackets() {
-        if (!isTls()) {
-            throw new NoSuchElementException("cannot get TLS Application Data packets for non-TLS TCP conversation");
-        }
-        return Collections.unmodifiableList(mTlsApplicationDataPackets);
-    }
-
-    /**
-     * Extracts the TCP sequence number from {@code packet} and adds it to the proper set of sequence numbers by
-     * analyzing the direction of the packet.
-     * @param packet A TCP packet (wrapped in a {@code PcapPacket}) that was added to this conversation and whose
-     *               sequence number is to be recorded as seen.
-     */
-    private void addSeqNumber(PcapPacket packet) {
-        // Note: below check is redundant if client code is correct as the call to check the precondition should already
-        // have been made by the addXPacket method that invokes this method. As such, the call below may be removed in
-        // favor of speed, but the improvement will be minor, hence the added safety may be worth it.
-        onAddPrecondition(packet);
-        // Extract sequence number.
-        int seqNo = packet.get(TcpPacket.class).getHeader().getSequenceNumber();
-        // Determine direction of packet and add packet's sequence number to corresponding set of sequence numbers.
-        switch (getDirection(packet)) {
-            case CLIENT_TO_SERVER:
-                // Client to server packet.
-                mSeqNumbersClient.add(seqNo);
-                break;
-            case SERVER_TO_CLIENT:
-                // Server to client packet.
-                mSeqNumbersSrv.add(seqNo);
-                break;
-            default:
-                throw new AssertionError(String.format("Unexpected value of enum '%s'",
-                        Direction.class.getSimpleName()));
-        }
-    }
-
-    /**
-     * Determine the direction of {@code packet}. An {@link IllegalArgumentException} is thrown if {@code packet} does
-     * not pertain to this conversation.
-     *
-     * @param packet The packet whose direction is to be determined.
-     * @return A {@link Direction} indicating the direction of the packet.
-     */
-    public Direction getDirection(PcapPacket packet) {
-        IpV4Packet ipPacket = packet.get(IpV4Packet.class);
-        String ipSrc = ipPacket.getHeader().getSrcAddr().getHostAddress();
-        String ipDst = ipPacket.getHeader().getDstAddr().getHostAddress();
-        // Determine direction of packet.
-        if (ipSrc.equals(mClientIp) && ipDst.equals(mServerIp)) {
-            // Client to server packet.
-            return Direction.CLIENT_TO_SERVER;
-        } else if (ipSrc.equals(mServerIp) && ipDst.equals(mClientIp)) {
-            // Server to client packet.
-            return Direction.SERVER_TO_CLIENT;
-        } else {
-            throw new IllegalArgumentException("getDirection: packet not related to " + getClass().getSimpleName());
-        }
-    }
-
-    /**
-     * Utility enum for expressing the direction of a packet pertaining to this {@code Conversation}.
-     */
-    public enum Direction {
-
-        CLIENT_TO_SERVER {
-            @Override
-            public String toCompactString() {
-                return "*";
-            }
-        },
-        SERVER_TO_CLIENT {
-            @Override
-            public String toCompactString() {
-                return "";
-            }
-        };
-
-        /**
-         * Get a compact string representation of this {@code Direction}.
-         * @return a compact string representation of this {@code Direction}.
-         */
-        abstract public String toCompactString();
-
-    }
-
-}
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FinAckPair.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FinAckPair.java
deleted file mode 100644 (file)
index d4451f3..0000000
+++ /dev/null
@@ -1,134 +0,0 @@
-package edu.uci.iotproject;
-
-import org.pcap4j.core.PcapPacket;
-import org.pcap4j.packet.IpV4Packet;
-import org.pcap4j.packet.TcpPacket;
-
-/**
- * Groups a FIN packet and its corresponding ACK packet. <b>Immutable and thread safe</b>.
- *
- * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
- * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
- */
-public class FinAckPair {
-
-    private final PcapPacket mFinPacket;
-    private final PcapPacket mCorrespondingAckPacket;
-
-    /**
-     * Constructs a {@code FinAckPair} given a FIN packet.
-     * The corresponding ACK packet field is set to {@code null}.
-     * @param finPacket A FIN packet.
-     */
-    public FinAckPair(PcapPacket finPacket) {
-        if (!finPacket.get(TcpPacket.class).getHeader().getFin()) {
-            throw new IllegalArgumentException("not a FIN packet");
-        }
-        mFinPacket = finPacket;
-        mCorrespondingAckPacket = null;
-    }
-
-    /**
-     * Constructs a {@code FinAckPair} given a FIN and an ACK packet.
-     * @param finPacket A FIN packet.
-     * @param correspondingAckPacket The ACK packet corresponding to {@code finPacket}.
-     */
-    public FinAckPair(PcapPacket finPacket, PcapPacket correspondingAckPacket) {
-        // Enforce class invariant, i.e. that the FIN and ACK are related.
-        // Note that it is indirectly checked whether finPacket is indeed a FIN packet
-        // as isCorrespondingAckPacket calls the single parameter constructor.
-        if (!FinAckPair.isCorrespondingAckPacket(finPacket, correspondingAckPacket)) {
-            throw new IllegalArgumentException("FIN and ACK not related");
-        }
-        mFinPacket = finPacket;
-        mCorrespondingAckPacket = correspondingAckPacket;
-    }
-
-    /**
-     * Get the FIN packet of this pair.
-     * @return the FIN packet of this pair.
-     */
-    public PcapPacket getFinPacket() {
-        return mFinPacket;
-    }
-
-    /**
-     * Get the corresponding ACK packet of this pair, if any.
-     * @return the corresponding ACK packet of this pair, if any.
-     */
-    public PcapPacket getCorrespondingAckPacket() {
-        return mCorrespondingAckPacket;
-    }
-
-    /**
-     * Was the FIN in this {@code FinAckPair} acknowledged?
-     *
-     * @return {@code true} if the corresponding ACK has been set in this {@code FinAckPair}.
-     */
-    public boolean isAcknowledged() {
-        return mFinPacket != null && mCorrespondingAckPacket != null;
-    }
-
-    /**
-     * Checks if a given packet is an ACK corresponding to the FIN packet in this {@code FinAckPair}.
-     * @return {@code true} if {@code packet} is an ACK that corresponds to the FIN in this pair, {@code false} otherwise.
-     */
-    public boolean isCorrespondingAckPacket(PcapPacket packet) {
-        IpV4Packet inputIpPacket = packet.get(IpV4Packet.class);
-        TcpPacket inputTcpPacket = packet.get(TcpPacket.class);
-        if (inputIpPacket == null || inputTcpPacket == null || !inputTcpPacket.getHeader().getAck()) {
-            return false;
-        }
-
-        IpV4Packet finIpPacket = mFinPacket.get(IpV4Packet.class);
-        TcpPacket finTcpPacket = mFinPacket.get(TcpPacket.class);
-
-        // Extract (srcIp:port,dstIp:port) for input and member (FIN) packets.
-        String inputPacketIpSrc = inputIpPacket.getHeader().getSrcAddr().getHostAddress();
-        String inputPacketIpDst = inputIpPacket.getHeader().getDstAddr().getHostAddress();
-        int inputPacketPortSrc = inputTcpPacket.getHeader().getSrcPort().valueAsInt();
-        int inputPacketPortDst = inputTcpPacket.getHeader().getDstPort().valueAsInt();
-        String finPacketIpSrc = finIpPacket.getHeader().getSrcAddr().getHostAddress();
-        String finPacketIpDst = finIpPacket.getHeader().getDstAddr().getHostAddress();
-        int finPacketPortSrc = finTcpPacket.getHeader().getSrcPort().valueAsInt();
-        int finPacketPortDst = finTcpPacket.getHeader().getDstPort().valueAsInt();
-
-        // For the two packets to be related, the dst of one must be the src of the other.
-        // Split into multiple if statements for readability. First check IP fields, then ports.
-        if (!(inputPacketIpDst.equals(finPacketIpSrc) && finPacketIpDst.equals(inputPacketIpSrc))) {
-            return false;
-        }
-        if (!(inputPacketPortDst == finPacketPortSrc && finPacketPortDst == inputPacketPortSrc)) {
-            return false;
-        }
-
-        // Packets are (most likely) related (part of same conversation/stream).
-        // Now all that is left for us to check is if the sequence numbers match.
-        // Note: recall that the FIN packet advances the seq numbers by 1,
-        // so the ACK number will be one larger than the seq. number in the FIN packet.
-        return inputTcpPacket.getHeader().getAcknowledgmentNumber() == finTcpPacket.getHeader().getSequenceNumber() + 1;
-    }
-
-    /**
-     * Static method to check if two given packets are a FIN and the corresponding ACK packet.
-     * The purpose of this method is a workaround to enforce the class invariant in the two parameter constructor.
-     * Specifically, the following should be avoided:
-     * <pre>
-     *     public FinAckPair(PcapPacket finPacket, PcapPacket correspondingAckPacket) {
-     *         mFinPacket = finPacket;
-     *         // Below line is considered bad practice as the object has not been fully initialized at this stage.
-     *         if (!this.isCorrespondingAckPacket(correspondingAckPacket)) {
-     *             // ... throw exception
-     *         }
-     *     }
-     * </pre>
-     * @param finPacket The FIN packet.
-     * @param ackPacket The ACK packet that is to be checked if it corresponds to the given FIN packet.
-     * @return {@code true} if the ACK corresponds to the FIN, {@code false} otherwise.
-     */
-    private static boolean isCorrespondingAckPacket(PcapPacket finPacket, PcapPacket ackPacket) {
-        FinAckPair tmp = new FinAckPair(finPacket);
-        return tmp.isCorrespondingAckPacket(ackPacket);
-    }
-
-}
index 1d0a9ba..c384852 100644 (file)
@@ -3,6 +3,7 @@ package edu.uci.iotproject;
 import edu.uci.iotproject.comparison.ComparisonFunctions;
 import edu.uci.iotproject.comparison.CompleteMatchPatternComparisonResult;
 import edu.uci.iotproject.comparison.PatternComparisonTask;
+import edu.uci.iotproject.trafficreassembly.layer3.Conversation;
 import org.pcap4j.core.NotOpenException;
 import org.pcap4j.core.PcapHandle;
 import org.pcap4j.core.PcapNativeException;
@@ -15,7 +16,6 @@ import java.io.*;
 import java.net.UnknownHostException;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
-import java.time.temporal.ChronoField;
 import java.util.*;
 import java.util.concurrent.*;
 
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/L2FlowReassembler.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/L2FlowReassembler.java
deleted file mode 100644 (file)
index b09189f..0000000
+++ /dev/null
@@ -1,84 +0,0 @@
-package edu.uci.iotproject;
-
-import org.pcap4j.core.PacketListener;
-import org.pcap4j.core.PcapPacket;
-import org.pcap4j.packet.EthernetPacket;
-import org.pcap4j.util.MacAddress;
-
-import java.util.*;
-
-/**
- * Reassembles traffic flows at layer 2, i.e., for each combination of hosts, creates a list of packets exchanged
- * between said hosts.
- *
- * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
- * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
- */
-public class L2FlowReassembler implements PacketListener {
-
-    /**
-     * Maps a pair of MAC addresses to the packets exchanged between the two hosts.
-     * The key is the concatenation of the two MAC addresses in hex string format, where the lexicographically smaller
-     * MAC is at the front of the string.
-     */
-    private final Map<String, Layer2Flow> mFlows = new HashMap<>();
-
-    private final List<Layer2FlowReassemblerObserver> mObservers = new ArrayList<>();
-
-    @Override
-    public void gotPacket(PcapPacket packet) {
-        // TODO: update to 802.11 packet...?
-        EthernetPacket ethPkt = packet.get(EthernetPacket.class);
-
-        MacAddress srcAddr = ethPkt.getHeader().getSrcAddr();
-        MacAddress dstAddr = ethPkt.getHeader().getDstAddr();
-
-        String key = keyFromAddresses(srcAddr, dstAddr);
-        // Create a new list if this pair of MAC addresses where not previously encountered and add packet to that list,
-        // or simply add to an existing list if one is present.
-        mFlows.computeIfAbsent(key, k -> {
-            Layer2Flow newFlow = new Layer2Flow(srcAddr, dstAddr);
-            // Inform observers of the new flow
-            mObservers.forEach(o -> o.onNewFlow(this, newFlow));
-            return newFlow;
-        }).addPacket(packet);
-    }
-
-    public void addObserver(Layer2FlowReassemblerObserver observer) {
-        mObservers.add(observer);
-    }
-
-    public void removeObserver(Layer2FlowReassemblerObserver observer) {
-        mObservers.remove(observer);
-    }
-
-    /**
-     * Get the traffic flow between two local endpoints ({@link MacAddress}es).
-     * @param addr1 The first endpoint.
-     * @param addr2 The second endpoint
-     * @return The traffic exchanged between the two endpoints.
-     */
-    public Layer2Flow getFlowForAddresses(MacAddress addr1, MacAddress addr2) {
-        return mFlows.get(keyFromAddresses(addr1, addr2));
-    }
-
-    /**
-     * Get all traffic flows, i.e., a traffic flow for each unique pair of endpoints (MAC addresses).
-     * @return All traffic flows.
-     */
-    public Collection<Layer2Flow> getFlows() {
-        return mFlows.values();
-    }
-
-    /**
-     * Given two {@link MacAddress}es, generates the corresponding key string used in {@link #mFlows}.
-     * @param addr1 The first address.
-     * @param addr2 The second address.
-     * @return the key string used in {@link #mFlows} corresponding to the two addresses.
-     */
-    private String keyFromAddresses(MacAddress addr1, MacAddress addr2) {
-        String addr1Str = addr1.toString();
-        String addr2Str = addr2.toString();
-        return addr1Str.compareTo(addr2Str) < 0 ? addr1Str + addr2Str : addr2Str + addr1Str;
-    }
-}
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Layer2Flow.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Layer2Flow.java
deleted file mode 100644 (file)
index d50a911..0000000
+++ /dev/null
@@ -1,79 +0,0 @@
-package edu.uci.iotproject;
-
-import edu.uci.iotproject.detection.Layer2FlowObserver;
-import org.pcap4j.core.PcapPacket;
-import org.pcap4j.packet.EthernetPacket;
-import org.pcap4j.util.MacAddress;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-/**
- * The packets exchanged between two endpoints (MAC addresses).
- *
- * @author Janus Varmarken
- */
-public class Layer2Flow {
-
-    private final MacAddress mEndpoint1;
-    private final MacAddress mEndpoint2;
-
-    private final List<Layer2FlowObserver> mFlowObservers = new ArrayList<>();
-
-    public Layer2Flow(MacAddress endpoint1, MacAddress endpoint2) {
-        mEndpoint1 = endpoint1;
-        mEndpoint2 = endpoint2;
-    }
-
-    public void addFlowObserver(Layer2FlowObserver observer) {
-        mFlowObservers.add(observer);
-    }
-
-    public void removeFlowObserver(Layer2FlowObserver observer) {
-        mFlowObservers.remove(observer);
-    }
-
-    /**
-     * The packets in the flow.
-     */
-    private final List<PcapPacket> mPackets = new ArrayList<>();
-
-    /**
-     * Add a packet to this flow.
-     * @param packet The packet that is to be added to the flow.
-     */
-    public void addPacket(PcapPacket packet) {
-        verifyAddresses(packet);
-        mPackets.add(packet);
-        // Notify flow observers of the new packet
-        mFlowObservers.forEach(o -> o.onNewPacket(this, packet));
-    }
-
-    public List<PcapPacket> getPackets() {
-        return Collections.unmodifiableList(mPackets);
-    }
-
-    private void verifyAddresses(PcapPacket packet) {
-        EthernetPacket ethPkt = packet.get(EthernetPacket.class);
-        MacAddress srcAddr = ethPkt.getHeader().getSrcAddr();
-        MacAddress dstAddr = ethPkt.getHeader().getDstAddr();
-        if ((mEndpoint1.equals(srcAddr) && mEndpoint2.equals(dstAddr)) ||
-                (mEndpoint1.equals(dstAddr) && mEndpoint2.equals(srcAddr))) {
-            // All is good.
-            return;
-        }
-        throw new IllegalArgumentException("Mismatch in MACs: packet does not pertain to this flow");
-    }
-
-}
-
-
-
-/*
-
-
- Packet stream -> flow reassembler -> flow1, flow2, flow3... -> for each flow, keep a sequence matcher for each sequence of cluster
-
-
- */
\ No newline at end of file
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Layer2FlowReassemblerObserver.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Layer2FlowReassemblerObserver.java
deleted file mode 100644 (file)
index c9eb2f7..0000000
+++ /dev/null
@@ -1,19 +0,0 @@
-package edu.uci.iotproject;
-
-/**
- * For observing a {@link L2FlowReassembler}.
- *
- * @author Janus Varmarken
- */
-public interface Layer2FlowReassemblerObserver {
-
-    /**
-     * Invoked when when a {@link L2FlowReassembler} detects a new flow (i.e., when it encounters traffic between two
-     * MAC addresses that has not previously communicated in the traffic trace).
-     *
-     * @param reassembler The {@link L2FlowReassembler} that detected the new flow.
-     * @param newFlow The new flow.
-     */
-    void onNewFlow(L2FlowReassembler reassembler, Layer2Flow newFlow);
-
-}
index 5b9a54e..a7e9a6c 100644 (file)
@@ -3,10 +3,9 @@ package edu.uci.iotproject;
 import static edu.uci.iotproject.analysis.UserAction.Type;
 
 import edu.uci.iotproject.analysis.*;
-import edu.uci.iotproject.comparison.seqalignment.ExtractedSequence;
-import edu.uci.iotproject.comparison.seqalignment.SequenceAlignment;
-import edu.uci.iotproject.comparison.seqalignment.SequenceExtraction;
 import edu.uci.iotproject.io.TriggerTimesFileReader;
+import edu.uci.iotproject.trafficreassembly.layer3.Conversation;
+import edu.uci.iotproject.trafficreassembly.layer3.TcpReassembler;
 import edu.uci.iotproject.util.PcapPacketUtils;
 import edu.uci.iotproject.util.PrintUtils;
 import org.apache.commons.math3.stat.clustering.Cluster;
@@ -15,8 +14,6 @@ import org.pcap4j.core.*;
 import org.pcap4j.packet.namednumber.DataLinkType;
 
 import java.io.EOFException;
-import java.io.File;
-import java.io.PrintWriter;
 import java.net.UnknownHostException;
 import java.time.Instant;
 import java.util.*;
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/TcpReassembler.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/TcpReassembler.java
deleted file mode 100644 (file)
index 72f0c1c..0000000
+++ /dev/null
@@ -1,260 +0,0 @@
-package edu.uci.iotproject;
-
-import org.pcap4j.core.PacketListener;
-import org.pcap4j.core.PcapPacket;
-import org.pcap4j.packet.*;
-
-import java.util.*;
-
-/**
- * Reassembles TCP conversations (streams).
- * <b>Note: current version only supports TCP over IPv4.</b>
- *
- * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
- * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
- */
-public class TcpReassembler implements PacketListener {
-
-    /**
-     * Holds <em>open</em> {@link Conversation}s, i.e., {@code Conversation}s that have <em>not</em> been detected as
-     * (gracefully) terminated based on the set of packets observed thus far.
-     * A {@link Conversation} is moved to {@link #mTerminatedConversations} if it can be determined that it is has
-     * terminated. Termination can be detected by a) observing two {@link FinAckPair}s, one in each direction, (graceful
-     * termination, see {@link Conversation#isGracefullyShutdown()}) or b) by observing a SYN packet that matches the
-     * four tuple of an existing {@code Conversation}, but which holds a <em>different</em> sequence number than the
-     * same-direction SYN packet recorded for the {@code Conversation}.
-     * <p>
-     * Note that due to limitations of the {@link Set} interface (specifically, there is no {@code get(T t)} method),
-     * we have to resort to a {@link Map} (in which keys map to themselves) to "mimic" a set with {@code get(T t)}
-     * functionality.
-     *
-     * @see <a href="https://stackoverflow.com/questions/7283338/getting-an-element-from-a-set">this question on StackOverflow.com</a>
-     */
-    private final Map<Conversation, Conversation> mOpenConversations = new HashMap<>();
-
-    /**
-     * Holds <em>terminated</em> {@link Conversation}s.
-     */
-    private final List<Conversation> mTerminatedConversations = new ArrayList<>();
-
-    @Override
-    public void gotPacket(PcapPacket pcapPacket) {
-        IpV4Packet ipPacket = pcapPacket.get(IpV4Packet.class);
-        TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class);
-
-        if (ipPacket == null || tcpPacket == null) {
-            return;
-        }
-        // ... TODO?
-        processPacket(pcapPacket);
-//        Class clazz = pcapPacket.getClass();
-//        RadiotapPacket radiotapPacket = pcapPacket.get(RadiotapPacket.class);
-//        Dot11ManagementPacket dot11ManagementPacket = pcapPacket.get(Dot11ManagementPacket.class);
-//        if (dot11ManagementPacket != null) {
-//            return;
-//        }
-//        if (radiotapPacket != null) {
-//            processRadiotapPacket(pcapPacket);
-//        }
-    }
-
-    /**
-     * Get the reassembled TCP connections. Note that if this is called while packets are still being processed (by
-     * calls to {@link #gotPacket(PcapPacket)}), the behavior is undefined and the returned list may be inconsistent.
-     * @return The reassembled TCP connections.
-     */
-    public List<Conversation> getTcpConversations() {
-        ArrayList<Conversation> combined = new ArrayList<>();
-        combined.addAll(mTerminatedConversations);
-        combined.addAll(mOpenConversations.values());
-        return combined;
-    }
-
-    private void processRadiotapPacket(PcapPacket pcapPacket) {
-        RadiotapPacket radiotapPacket = pcapPacket.get(RadiotapPacket.class);
-
-        RadiotapPacket.RadiotapHeader header = radiotapPacket.getHeader();
-        short length = header.getLength();
-        ArrayList<RadiotapPacket.RadiotapData> radiotapData = header.getDataFields();
-        // TODO: We can handle this 802.11 QoS data by creating our own class
-        // TODO: We only need to handle the first few bytes for source, destination, receiver, and transmitter
-        // TODO: addresses
-        Packet dataPacket = radiotapPacket.getPayload();
-        int dataLength = dataPacket.length();
-    }
-
-    private void processPacket(PcapPacket pcapPacket) {
-        TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class);
-        // Handle client connection initiation attempts.
-        if (tcpPacket.getHeader().getSyn() && !tcpPacket.getHeader().getAck()) {
-            // A segment with the SYN flag set, but no ACK flag indicates that a client is attempting to initiate a new
-            // connection.
-            processNewConnectionRequest(pcapPacket);
-            return;
-        }
-        // Handle server connection initiation acknowledgement
-        if (tcpPacket.getHeader().getSyn() && tcpPacket.getHeader().getAck()) {
-            // A segment with both the SYN and ACK flags set indicates that the server has accepted the client's request
-            // to initiate a new connection.
-            processNewConnectionAck(pcapPacket);
-            return;
-        }
-        // Handle resets
-        if (tcpPacket.getHeader().getRst()) {
-            processRstPacket(pcapPacket);
-            return;
-        }
-        // Handle FINs
-        if (tcpPacket.getHeader().getFin()) {
-            // Handle FIN packet.
-            processFinPacket(pcapPacket);
-        }
-        // Handle ACKs (currently only ACKs of FINS)
-        if (tcpPacket.getHeader().getAck()) {
-            processAck(pcapPacket);
-        }
-        // Handle packets that carry payload (application data).
-        if (tcpPacket.getPayload() != null) {
-            processPayloadPacket(pcapPacket);
-        }
-    }
-
-    private void processNewConnectionRequest(PcapPacket clientSynPacket) {
-        // A SYN w/o ACK always originates from the client.
-        Conversation conv = Conversation.fromPcapPacket(clientSynPacket, true);
-        conv.addSynPacket(clientSynPacket);
-        // Is there an ongoing conversation for the same four tuple (clientIp, clientPort, serverIp, serverPort) as
-        // found in the new SYN packet?
-        Conversation ongoingConv = mOpenConversations.get(conv);
-        if (ongoingConv != null) {
-            if (ongoingConv.isRetransmission(clientSynPacket)) {
-                // SYN retransmission detected, do nothing.
-                return;
-                // TODO: the way retransmission detection is implemented may cause a bug for connections where we have
-                // not recorded the initial SYN, but only the SYN ACK, as retransmission is determined by comparing the
-                // sequence numbers of initial SYNs -- and if no initial SYN is present for the Conversation, the new
-                // SYN will be interpreted as a retransmission. Possible fix: let isRentransmission ALWAYS return false
-                // when presented with a SYN packet when the Conversation already holds a SYN ACK packet?
-            } else {
-                // New SYN has different sequence number than SYN recorded for ongoingConv, so this must be an attempt
-                // to establish a new conversation with the same four tuple as ongoingConv.
-                // Mark existing connection as terminated.
-                // TODO: is this 100% theoretically correct, e.g., if many connection attempts are made back to back? And RST packets?
-                mTerminatedConversations.add(ongoingConv);
-                mOpenConversations.remove(ongoingConv);
-            }
-        }
-        // Finally, update the map of open connections with the new connection.
-        mOpenConversations.put(conv, conv);
-    }
-
-
-    /*
-     * TODO a problem across the board for all processXPacket methods below:
-     * if we start the capture in the middle of a TCP connection, we will not have an entry for the conversation in the
-     * map as we have not seen the initial SYN packet.
-     * Two ways we can address this:
-     * a) Perform null-checks and ignore packets for which we have not seen SYN
-     *    + easy to get correct
-     *    - we discard data (issue for long-lived connections!)
-     * b) Add a corresponding conversation entry whenever we encounter a packet that does not map to a conversation
-     *    + we consider all data
-     *    - not immediately clear if this will introduce bugs (incorrectly mapping packets to wrong conversations?)
-     *
-     *  [[[ I went with option b) for now; see getOngoingConversationOrCreateNew(PcapPacket pcapPacket). ]]]
-     */
-
-    private void processNewConnectionAck(PcapPacket srvSynPacket) {
-        // Find the corresponding ongoing connection, if any (if we start the capture just *after* the initial SYN, no
-        // ongoing conversation entry will exist, so it must be created in that case).
-//        Conversation conv = mOpenConversations.get(Conversation.fromPcapPacket(srvSynPacket, false));
-        Conversation conv = getOngoingConversationOrCreateNew(srvSynPacket);
-        // Note: exploits &&'s short-circuit operation: only attempts to add non-retransmissions.
-        if (!conv.isRetransmission(srvSynPacket) && !conv.addSynPacket(srvSynPacket)) {
-            // For safety/debugging: if NOT a retransmission and add fails,
-            // something has gone terribly wrong/invariant is broken.
-//            throw new AssertionError("Attempt to add SYN ACK packet that was NOT a retransmission failed." +
-//                    Conversation.class.getSimpleName() + " invariant broken.");
-        }
-    }
-
-    private void processRstPacket(PcapPacket rstPacket) {
-        Conversation conv = getOngoingConversationOrCreateNew(rstPacket);
-        // Add RST packet to conversation.
-        conv.addRstPacket(rstPacket);
-        // Move conversation to set of terminated conversations.
-        mTerminatedConversations.add(conv);
-        mOpenConversations.remove(conv, conv);
-    }
-
-    private void processFinPacket(PcapPacket finPacket) {
-//        getOngoingConversationForPacket(finPacket).addFinPacket(finPacket);
-        getOngoingConversationOrCreateNew(finPacket).addFinPacket(finPacket);
-    }
-
-    private void processAck(PcapPacket ackPacket) {
-//        getOngoingConversationForPacket(ackPacket).attemptAcknowledgementOfFin(ackPacket);
-        // Note that unlike the style for SYN, FIN, and payload packets, for "ACK only" packets, we want to avoid
-        // creating a new conversation.
-        Conversation conv = getOngoingConversationForPacket(ackPacket);
-        if (conv != null) {
-            // The ACK may be an ACK of a FIN, so attempt to mark the FIN as ack'ed.
-            conv.attemptAcknowledgementOfFin(ackPacket);
-            if (conv.isGracefullyShutdown()) {
-                // Move conversation to set of terminated conversations.
-                mTerminatedConversations.add(conv);
-                mOpenConversations.remove(conv);
-            }
-        }
-        // Note: add (additional) processing of ACKs (that are not ACKs of FINs) as necessary here...
-    }
-
-    private void processPayloadPacket(PcapPacket pcapPacket) {
-//        getOngoingConversationForPacket(pcapPacket).addPacket(pcapPacket, true);
-        getOngoingConversationOrCreateNew(pcapPacket).addPacket(pcapPacket, true);
-    }
-
-    /**
-     * Locates an ongoing conversation (if any) that {@code pcapPacket} pertains to.
-     * @param pcapPacket The packet that is to be mapped to an ongoing {@code Conversation}.
-     * @return The {@code Conversation} matching {@code pcapPacket} or {@code null} if there is no match.
-     */
-    private Conversation getOngoingConversationForPacket(PcapPacket pcapPacket) {
-        // We cannot know if this is a client-to-server or server-to-client packet without trying both options...
-        Conversation conv = mOpenConversations.get(Conversation.fromPcapPacket(pcapPacket, true));
-        if (conv == null) {
-            conv = mOpenConversations.get(Conversation.fromPcapPacket(pcapPacket, false));
-        }
-        return conv;
-    }
-
-    /**
-     * Like {@link #getOngoingConversationForPacket(PcapPacket)}, but creates and inserts a new {@code Conversation}
-     * into {@link #mOpenConversations} if no open conversation is found (i.e., in the case that
-     * {@link #getOngoingConversationForPacket(PcapPacket)} returns {@code null}).
-     *
-     * @param pcapPacket The packet that is to be mapped to an ongoing {@code Conversation}.
-     * @return The existing, ongoing {@code Conversation} matching {@code pcapPacket} or the newly created one in case
-     *         no match was found.
-     */
-    private Conversation getOngoingConversationOrCreateNew(PcapPacket pcapPacket) {
-        Conversation conv = getOngoingConversationForPacket(pcapPacket);
-        if (conv == null) {
-            TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class);
-            if (tcpPacket.getHeader().getSyn() && tcpPacket.getHeader().getAck()) {
-                // A SYN ACK packet always originates from the server (it is a reply to the initial SYN packet from the client)
-                conv = Conversation.fromPcapPacket(pcapPacket, false);
-            } else {
-                // TODO: can we do anything else but arbitrarily select who is designated as the server in this case?
-                // We can check if the IP prefix matches a local IP when handling traffic observed inside the local
-                // network, but that obviously won't be a useful strategy for an observer at the WAN port.
-                String srcIp = pcapPacket.get(IpV4Packet.class).getHeader().getSrcAddr().getHostAddress();
-                // TODO: REPLACE THE ROUTER'S IP WITH A PARAMETER!!!
-                boolean clientIsSrc = srcIp.startsWith("10.") || srcIp.startsWith("192.168.") || srcIp.equals("128.195.205.105");
-                conv = Conversation.fromPcapPacket(pcapPacket, clientIsSrc);
-            }
-            mOpenConversations.put(conv, conv);
-        }
-        return conv;
-    }
-}
index 7484d8a..f1c264b 100644 (file)
@@ -1,6 +1,6 @@
 package edu.uci.iotproject.analysis;
 
-import edu.uci.iotproject.Conversation;
+import edu.uci.iotproject.trafficreassembly.layer3.Conversation;
 import edu.uci.iotproject.DnsMap;
 import edu.uci.iotproject.util.PcapPacketUtils;
 import org.pcap4j.core.PcapPacket;
index ebc87b1..983de12 100644 (file)
@@ -1,8 +1,8 @@
 package edu.uci.iotproject.analysis;
 
-import edu.uci.iotproject.Conversation;
+import edu.uci.iotproject.trafficreassembly.layer3.Conversation;
 import edu.uci.iotproject.DnsMap;
-import edu.uci.iotproject.TcpReassembler;
+import edu.uci.iotproject.trafficreassembly.layer3.TcpReassembler;
 import org.pcap4j.core.PacketListener;
 import org.pcap4j.core.PcapPacket;
 
index 15eda20..960d6a7 100644 (file)
@@ -1,6 +1,6 @@
 package edu.uci.iotproject.comparison;
 
-import edu.uci.iotproject.Conversation;
+import edu.uci.iotproject.trafficreassembly.layer3.Conversation;
 import edu.uci.iotproject.FlowPattern;
 import org.pcap4j.core.PcapPacket;
 import org.pcap4j.packet.TcpPacket;
index 808dee7..5a888b7 100644 (file)
@@ -1,6 +1,6 @@
 package edu.uci.iotproject.comparison;
 
-import edu.uci.iotproject.Conversation;
+import edu.uci.iotproject.trafficreassembly.layer3.Conversation;
 import edu.uci.iotproject.FlowPattern;
 
 import java.util.concurrent.Callable;
index c611968..6aaa318 100644 (file)
@@ -1,9 +1,8 @@
 package edu.uci.iotproject.comparison.seqalignment;
 
-import edu.uci.iotproject.Conversation;
+import edu.uci.iotproject.trafficreassembly.layer3.Conversation;
 import edu.uci.iotproject.analysis.TcpConversationUtils;
 
-import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
index 25f3191..1ff2c02 100644 (file)
@@ -1,7 +1,7 @@
 package edu.uci.iotproject.detection;
 
-import edu.uci.iotproject.Conversation;
-import edu.uci.iotproject.TcpReassembler;
+import edu.uci.iotproject.trafficreassembly.layer3.Conversation;
+import edu.uci.iotproject.trafficreassembly.layer3.TcpReassembler;
 import edu.uci.iotproject.analysis.TcpConversationUtils;
 import edu.uci.iotproject.io.PcapHandleReader;
 import edu.uci.iotproject.util.PrintUtils;
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/Layer2ClusterMatcher.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/Layer2ClusterMatcher.java
deleted file mode 100644 (file)
index e0438d6..0000000
+++ /dev/null
@@ -1,148 +0,0 @@
-package edu.uci.iotproject.detection;
-
-import edu.uci.iotproject.L2FlowReassembler;
-import edu.uci.iotproject.Layer2Flow;
-import edu.uci.iotproject.Layer2FlowReassemblerObserver;
-import edu.uci.iotproject.io.PcapHandleReader;
-import edu.uci.iotproject.util.PrintUtils;
-import org.pcap4j.core.*;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * TODO add class documentation.
- *
- * @author Janus Varmarken
- */
-public class Layer2ClusterMatcher extends AbstractClusterMatcher implements Layer2FlowReassemblerObserver, Layer2FlowObserver {
-
-    public static void main(String[] args) throws PcapNativeException, NotOpenException {
-        final String onSignatureFile = "/Users/varmarken/temp/UCI IoT Project/experiments/training/signatures/tplink-plug/tplink-plug-onSignature-device-side.sig";
-        List<List<List<PcapPacket>>> onSignature = PrintUtils.deserializeSignatureFromFile(onSignatureFile);
-
-
-        L2FlowReassembler flowReassembler = new L2FlowReassembler();
-
-        Layer2ClusterMatcher l2ClusterMatcher = new Layer2ClusterMatcher(onSignature.get(0));
-        flowReassembler.addObserver(l2ClusterMatcher);
-
-        final String inputPcapFile = "/Users/varmarken/temp/UCI IoT Project/experiments/2018-07/tplink/tplink.wlan1.local.pcap";
-
-        PcapHandle handle;
-        try {
-            handle = Pcaps.openOffline(inputPcapFile, PcapHandle.TimestampPrecision.NANO);
-        } catch (PcapNativeException pne) {
-            handle = Pcaps.openOffline(inputPcapFile);
-        }
-        PcapHandleReader reader = new PcapHandleReader(handle, p -> true, flowReassembler);
-        reader.readFromHandle();
-
-
-    }
-
-
-    private final List<Layer2SequenceMatcher> mSeqMatchers;
-
-    public Layer2ClusterMatcher(List<List<PcapPacket>> cluster) {
-        super(cluster);
-        // Setup a sequence matcher for each sequence of the pruned cluster
-        mSeqMatchers = new ArrayList<>();
-        mCluster.forEach(seq -> mSeqMatchers.add(new Layer2SequenceMatcher(seq)));
-
-//        for (int i = 0; i < mCluster.size(); i++) {
-//
-//
-//            mSeqMatchers[i] = new Layer2SequenceMatcher(mCluster.get(i));
-//
-//
-//        }
-    }
-
-//    @Override
-//    public void gotPacket(PcapPacket packet) {
-//        // Forward the packet to all sequence matchers.
-//        for (Layer2SequenceMatcher matcher : mSeqMatchers) {
-//            matcher.gotPacket(packet);
-//        }
-//
-//
-//    }
-
-
-    private final Map<Layer2Flow, List<Layer2SequenceMatcher>> mPerFlowSeqMatchers = new HashMap<>();
-
-    @Override
-    public void onNewPacket(Layer2Flow flow, PcapPacket newPacket) {
-        if (mPerFlowSeqMatchers.get(flow) == null) {
-            // If this is the first time we encounter this flow, we need to set up sequence matchers for it.
-            List<Layer2SequenceMatcher> matchers = new ArrayList<>();
-            mCluster.forEach(seq -> matchers.add(new Layer2SequenceMatcher(seq)));
-            mPerFlowSeqMatchers.put(flow, matchers);
-        }
-        // Buffer for new sequence matchers that will take over the job of observing for the first packet when a
-        // sequence matcher advances beyond the first packet.
-        List<Layer2SequenceMatcher> newSeqMatchers = new ArrayList<>();
-        // Buffer for sequence matchers that have terminated and are to be removed from mPerFlowSeqMatchers.
-        List<Layer2SequenceMatcher> terminatedSeqMatchers = new ArrayList<>();
-        // Present the new packet to all sequence matchers
-        for (Layer2SequenceMatcher sm : mPerFlowSeqMatchers.get(flow)) {
-            boolean matched = sm.matchPacket(newPacket);
-            if (matched && sm.getMatchedPacketsCount() == 1) {
-                // Setup a new sequence matcher that matches from the beginning of the sequence so as to keep
-                // progressing in the sequence matcher that just matched the current packet, while still allowing
-                // for matches of the full sequence in later traffic. This is to accommodate the case where the
-                // first packet of a sequence is detected in an early packet, but where the remaining packets of
-                // that sequence do not appear until way later in time (e.g., if the first packet of the sequence
-                // by chance is generated from traffic unrelated to the trigger traffic).
-                // Note that we must store the new sequence matcher in a buffer and add it outside the loop in order to
-                // prevent concurrent modification exceptions.
-                newSeqMatchers.add(new Layer2SequenceMatcher(sm.getTargetSequence()));
-            }
-            if (matched && sm.getMatchedPacketsCount() == sm.getTargetSequencePacketCount()) {
-                // This sequence matcher has a match of the sequence it was searching for
-                // TODO report it.... for now just do a dummy printout.
-                System.out.println("SEQUENCE MATCHER HAS A MATCH AT " + sm.getMatchedPackets().get(0).getTimestamp());
-                // Mark the sequence matcher for removal. No need to create a replacement one as we do that whenever the
-                // first packet of the sequence is matched (see above).
-                terminatedSeqMatchers.add(sm);
-            }
-        }
-        // Add the new sequence matchers, if any.
-        mPerFlowSeqMatchers.get(flow).addAll(newSeqMatchers);
-        // Remove the terminated sequence matchers, if any.
-        mPerFlowSeqMatchers.get(flow).removeAll(terminatedSeqMatchers);
-    }
-
-
-    @Override
-    protected List<List<PcapPacket>> pruneCluster(List<List<PcapPacket>> cluster) {
-        // Note: we assume that all sequences in the input cluster are of the same length and that their packet
-        // directions are identical.
-        List<List<PcapPacket>> prunedCluster = new ArrayList<>();
-        for (List<PcapPacket> originalClusterSeq : cluster) {
-            boolean alreadyPresent = prunedCluster.stream().anyMatch(pcPkts -> {
-                for (int i = 0; i < pcPkts.size(); i++) {
-                    if (pcPkts.get(i).getOriginalLength() != originalClusterSeq.get(i).getOriginalLength()) {
-                        return false;
-                    }
-                }
-                return true;
-            });
-            if (!alreadyPresent) {
-                // Add the sequence if not already present in the pruned cluster.
-                prunedCluster.add(originalClusterSeq);
-            }
-        }
-        return prunedCluster;
-    }
-
-
-    @Override
-    public void onNewFlow(L2FlowReassembler reassembler, Layer2Flow newFlow) {
-        // Subscribe to the new flow to get updates whenever a new packet pertaining to the flow is processed.
-        newFlow.addFlowObserver(this);
-    }
-}
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/Layer2SequenceMatcher.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/Layer2SequenceMatcher.java
deleted file mode 100644 (file)
index 35d628d..0000000
+++ /dev/null
@@ -1,76 +0,0 @@
-package edu.uci.iotproject.detection;
-
-import edu.uci.iotproject.analysis.TriggerTrafficExtractor;
-import org.pcap4j.core.PcapPacket;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * TODO add class documentation.
- *
- * @author Janus Varmarken
- */
-public class Layer2SequenceMatcher {
-
-    /**
-     * The sequence this {@link Layer2SequenceMatcher} is searching for.
-     */
-    private final List<PcapPacket> mSequence;
-
-    /**
-     * Buffer of actual packets seen so far that match the searched sequence (i.e., constitutes a subsequence of the
-     * searched sequence).
-     */
-    private final List<PcapPacket> mMatchedPackets = new ArrayList<>();
-
-    public Layer2SequenceMatcher(List<PcapPacket> sequence) {
-        mSequence = sequence;
-    }
-
-    public boolean matchPacket(PcapPacket packet) {
-        // The packet we want to match next.
-        PcapPacket expected = mSequence.get(mMatchedPackets.size());
-        // First verify if the received packet has the length we're looking for.
-        if (packet.getOriginalLength() == expected.getOriginalLength()) {
-            // Next apply timing constraints:
-            // - to be a match, the packet must have a later timestamp than any other packet currently matched
-            // - does adding the packet cause the max allowed time between first packet and last packet to be exceeded?
-            if (mMatchedPackets.size() > 0 &&
-                    !packet.getTimestamp().isAfter(mMatchedPackets.get(mMatchedPackets.size()-1).getTimestamp())) {
-                return false;
-            }
-            if (mMatchedPackets.size() > 0 &&
-                    packet.getTimestamp().
-                            isAfter(mMatchedPackets.get(0).getTimestamp().
-                                    plusMillis(TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS))) {
-                // Packet too
-                return false;
-            }
-            // TODO (how to) check directions?
-            // This packet has a length matching next packet of searched sequence, so we store it and advance.
-            mMatchedPackets.add(packet);
-            if (mMatchedPackets.size() == mSequence.size()) {
-                // TODO report (to observers?) that we are done.
-            }
-            return true;
-        }
-        return false;
-    }
-
-    public int getMatchedPacketsCount() {
-        return mMatchedPackets.size();
-    }
-
-    public int getTargetSequencePacketCount() {
-        return mSequence.size();
-    }
-
-    public List<PcapPacket> getTargetSequence() {
-        return mSequence;
-    }
-
-    public List<PcapPacket> getMatchedPackets() {
-        return mMatchedPackets;
-    }
-}
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer2/Layer2ClusterMatcher.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer2/Layer2ClusterMatcher.java
new file mode 100644 (file)
index 0000000..d50a517
--- /dev/null
@@ -0,0 +1,150 @@
+package edu.uci.iotproject.detection.layer2;
+
+import edu.uci.iotproject.trafficreassembly.layer2.Layer2FlowReassembler;
+import edu.uci.iotproject.trafficreassembly.layer2.Layer2Flow;
+import edu.uci.iotproject.trafficreassembly.layer2.Layer2FlowReassemblerObserver;
+import edu.uci.iotproject.detection.AbstractClusterMatcher;
+import edu.uci.iotproject.detection.Layer2FlowObserver;
+import edu.uci.iotproject.io.PcapHandleReader;
+import edu.uci.iotproject.util.PrintUtils;
+import org.pcap4j.core.*;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * TODO add class documentation.
+ *
+ * @author Janus Varmarken
+ */
+public class Layer2ClusterMatcher extends AbstractClusterMatcher implements Layer2FlowReassemblerObserver, Layer2FlowObserver {
+
+    public static void main(String[] args) throws PcapNativeException, NotOpenException {
+        final String onSignatureFile = "/Users/varmarken/temp/UCI IoT Project/experiments/training/signatures/tplink-plug/tplink-plug-onSignature-device-side.sig";
+        List<List<List<PcapPacket>>> onSignature = PrintUtils.deserializeSignatureFromFile(onSignatureFile);
+
+
+        Layer2FlowReassembler flowReassembler = new Layer2FlowReassembler();
+
+        Layer2ClusterMatcher l2ClusterMatcher = new Layer2ClusterMatcher(onSignature.get(0));
+        flowReassembler.addObserver(l2ClusterMatcher);
+
+        final String inputPcapFile = "/Users/varmarken/temp/UCI IoT Project/experiments/2018-07/tplink/tplink.wlan1.local.pcap";
+
+        PcapHandle handle;
+        try {
+            handle = Pcaps.openOffline(inputPcapFile, PcapHandle.TimestampPrecision.NANO);
+        } catch (PcapNativeException pne) {
+            handle = Pcaps.openOffline(inputPcapFile);
+        }
+        PcapHandleReader reader = new PcapHandleReader(handle, p -> true, flowReassembler);
+        reader.readFromHandle();
+
+
+    }
+
+
+    private final List<Layer2SequenceMatcher> mSeqMatchers;
+
+    public Layer2ClusterMatcher(List<List<PcapPacket>> cluster) {
+        super(cluster);
+        // Setup a sequence matcher for each sequence of the pruned cluster
+        mSeqMatchers = new ArrayList<>();
+        mCluster.forEach(seq -> mSeqMatchers.add(new Layer2SequenceMatcher(seq)));
+
+//        for (int i = 0; i < mCluster.size(); i++) {
+//
+//
+//            mSeqMatchers[i] = new Layer2SequenceMatcher(mCluster.get(i));
+//
+//
+//        }
+    }
+
+//    @Override
+//    public void gotPacket(PcapPacket packet) {
+//        // Forward the packet to all sequence matchers.
+//        for (Layer2SequenceMatcher matcher : mSeqMatchers) {
+//            matcher.gotPacket(packet);
+//        }
+//
+//
+//    }
+
+
+    private final Map<Layer2Flow, List<Layer2SequenceMatcher>> mPerFlowSeqMatchers = new HashMap<>();
+
+    @Override
+    public void onNewPacket(Layer2Flow flow, PcapPacket newPacket) {
+        if (mPerFlowSeqMatchers.get(flow) == null) {
+            // If this is the first time we encounter this flow, we need to set up sequence matchers for it.
+            List<Layer2SequenceMatcher> matchers = new ArrayList<>();
+            mCluster.forEach(seq -> matchers.add(new Layer2SequenceMatcher(seq)));
+            mPerFlowSeqMatchers.put(flow, matchers);
+        }
+        // Buffer for new sequence matchers that will take over the job of observing for the first packet when a
+        // sequence matcher advances beyond the first packet.
+        List<Layer2SequenceMatcher> newSeqMatchers = new ArrayList<>();
+        // Buffer for sequence matchers that have terminated and are to be removed from mPerFlowSeqMatchers.
+        List<Layer2SequenceMatcher> terminatedSeqMatchers = new ArrayList<>();
+        // Present the new packet to all sequence matchers
+        for (Layer2SequenceMatcher sm : mPerFlowSeqMatchers.get(flow)) {
+            boolean matched = sm.matchPacket(newPacket);
+            if (matched && sm.getMatchedPacketsCount() == 1) {
+                // Setup a new sequence matcher that matches from the beginning of the sequence so as to keep
+                // progressing in the sequence matcher that just matched the current packet, while still allowing
+                // for matches of the full sequence in later traffic. This is to accommodate the case where the
+                // first packet of a sequence is detected in an early packet, but where the remaining packets of
+                // that sequence do not appear until way later in time (e.g., if the first packet of the sequence
+                // by chance is generated from traffic unrelated to the trigger traffic).
+                // Note that we must store the new sequence matcher in a buffer and add it outside the loop in order to
+                // prevent concurrent modification exceptions.
+                newSeqMatchers.add(new Layer2SequenceMatcher(sm.getTargetSequence()));
+            }
+            if (matched && sm.getMatchedPacketsCount() == sm.getTargetSequencePacketCount()) {
+                // This sequence matcher has a match of the sequence it was searching for
+                // TODO report it.... for now just do a dummy printout.
+                System.out.println("SEQUENCE MATCHER HAS A MATCH AT " + sm.getMatchedPackets().get(0).getTimestamp());
+                // Mark the sequence matcher for removal. No need to create a replacement one as we do that whenever the
+                // first packet of the sequence is matched (see above).
+                terminatedSeqMatchers.add(sm);
+            }
+        }
+        // Add the new sequence matchers, if any.
+        mPerFlowSeqMatchers.get(flow).addAll(newSeqMatchers);
+        // Remove the terminated sequence matchers, if any.
+        mPerFlowSeqMatchers.get(flow).removeAll(terminatedSeqMatchers);
+    }
+
+
+    @Override
+    protected List<List<PcapPacket>> pruneCluster(List<List<PcapPacket>> cluster) {
+        // Note: we assume that all sequences in the input cluster are of the same length and that their packet
+        // directions are identical.
+        List<List<PcapPacket>> prunedCluster = new ArrayList<>();
+        for (List<PcapPacket> originalClusterSeq : cluster) {
+            boolean alreadyPresent = prunedCluster.stream().anyMatch(pcPkts -> {
+                for (int i = 0; i < pcPkts.size(); i++) {
+                    if (pcPkts.get(i).getOriginalLength() != originalClusterSeq.get(i).getOriginalLength()) {
+                        return false;
+                    }
+                }
+                return true;
+            });
+            if (!alreadyPresent) {
+                // Add the sequence if not already present in the pruned cluster.
+                prunedCluster.add(originalClusterSeq);
+            }
+        }
+        return prunedCluster;
+    }
+
+
+    @Override
+    public void onNewFlow(Layer2FlowReassembler reassembler, Layer2Flow newFlow) {
+        // Subscribe to the new flow to get updates whenever a new packet pertaining to the flow is processed.
+        newFlow.addFlowObserver(this);
+    }
+}
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SequenceMatcher.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SequenceMatcher.java
new file mode 100644 (file)
index 0000000..de5c14a
--- /dev/null
@@ -0,0 +1,76 @@
+package edu.uci.iotproject.detection.layer2;
+
+import edu.uci.iotproject.analysis.TriggerTrafficExtractor;
+import org.pcap4j.core.PcapPacket;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * TODO add class documentation.
+ *
+ * @author Janus Varmarken
+ */
+public class Layer2SequenceMatcher {
+
+    /**
+     * The sequence this {@link Layer2SequenceMatcher} is searching for.
+     */
+    private final List<PcapPacket> mSequence;
+
+    /**
+     * Buffer of actual packets seen so far that match the searched sequence (i.e., constitutes a subsequence of the
+     * searched sequence).
+     */
+    private final List<PcapPacket> mMatchedPackets = new ArrayList<>();
+
+    public Layer2SequenceMatcher(List<PcapPacket> sequence) {
+        mSequence = sequence;
+    }
+
+    public boolean matchPacket(PcapPacket packet) {
+        // The packet we want to match next.
+        PcapPacket expected = mSequence.get(mMatchedPackets.size());
+        // First verify if the received packet has the length we're looking for.
+        if (packet.getOriginalLength() == expected.getOriginalLength()) {
+            // Next apply timing constraints:
+            // - to be a match, the packet must have a later timestamp than any other packet currently matched
+            // - does adding the packet cause the max allowed time between first packet and last packet to be exceeded?
+            if (mMatchedPackets.size() > 0 &&
+                    !packet.getTimestamp().isAfter(mMatchedPackets.get(mMatchedPackets.size()-1).getTimestamp())) {
+                return false;
+            }
+            if (mMatchedPackets.size() > 0 &&
+                    packet.getTimestamp().
+                            isAfter(mMatchedPackets.get(0).getTimestamp().
+                                    plusMillis(TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS))) {
+                // Packet too
+                return false;
+            }
+            // TODO (how to) check directions?
+            // This packet has a length matching next packet of searched sequence, so we store it and advance.
+            mMatchedPackets.add(packet);
+            if (mMatchedPackets.size() == mSequence.size()) {
+                // TODO report (to observers?) that we are done.
+            }
+            return true;
+        }
+        return false;
+    }
+
+    public int getMatchedPacketsCount() {
+        return mMatchedPackets.size();
+    }
+
+    public int getTargetSequencePacketCount() {
+        return mSequence.size();
+    }
+
+    public List<PcapPacket> getTargetSequence() {
+        return mSequence;
+    }
+
+    public List<PcapPacket> getMatchedPackets() {
+        return mMatchedPackets;
+    }
+}
index 9cecf56..060387a 100644 (file)
@@ -1,7 +1,7 @@
 package edu.uci.iotproject.evaluation;
 
-import edu.uci.iotproject.Conversation;
-import edu.uci.iotproject.TcpReassembler;
+import edu.uci.iotproject.trafficreassembly.layer3.Conversation;
+import edu.uci.iotproject.trafficreassembly.layer3.TcpReassembler;
 import edu.uci.iotproject.io.PcapHandleReader;
 import edu.uci.iotproject.util.PrintUtils;
 import org.pcap4j.core.*;
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2Flow.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2Flow.java
new file mode 100644 (file)
index 0000000..f1b7190
--- /dev/null
@@ -0,0 +1,79 @@
+package edu.uci.iotproject.trafficreassembly.layer2;
+
+import edu.uci.iotproject.detection.Layer2FlowObserver;
+import org.pcap4j.core.PcapPacket;
+import org.pcap4j.packet.EthernetPacket;
+import org.pcap4j.util.MacAddress;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * The packets exchanged between two endpoints (MAC addresses).
+ *
+ * @author Janus Varmarken
+ */
+public class Layer2Flow {
+
+    private final MacAddress mEndpoint1;
+    private final MacAddress mEndpoint2;
+
+    private final List<Layer2FlowObserver> mFlowObservers = new ArrayList<>();
+
+    public Layer2Flow(MacAddress endpoint1, MacAddress endpoint2) {
+        mEndpoint1 = endpoint1;
+        mEndpoint2 = endpoint2;
+    }
+
+    public void addFlowObserver(Layer2FlowObserver observer) {
+        mFlowObservers.add(observer);
+    }
+
+    public void removeFlowObserver(Layer2FlowObserver observer) {
+        mFlowObservers.remove(observer);
+    }
+
+    /**
+     * The packets in the flow.
+     */
+    private final List<PcapPacket> mPackets = new ArrayList<>();
+
+    /**
+     * Add a packet to this flow.
+     * @param packet The packet that is to be added to the flow.
+     */
+    public void addPacket(PcapPacket packet) {
+        verifyAddresses(packet);
+        mPackets.add(packet);
+        // Notify flow observers of the new packet
+        mFlowObservers.forEach(o -> o.onNewPacket(this, packet));
+    }
+
+    public List<PcapPacket> getPackets() {
+        return Collections.unmodifiableList(mPackets);
+    }
+
+    private void verifyAddresses(PcapPacket packet) {
+        EthernetPacket ethPkt = packet.get(EthernetPacket.class);
+        MacAddress srcAddr = ethPkt.getHeader().getSrcAddr();
+        MacAddress dstAddr = ethPkt.getHeader().getDstAddr();
+        if ((mEndpoint1.equals(srcAddr) && mEndpoint2.equals(dstAddr)) ||
+                (mEndpoint1.equals(dstAddr) && mEndpoint2.equals(srcAddr))) {
+            // All is good.
+            return;
+        }
+        throw new IllegalArgumentException("Mismatch in MACs: packet does not pertain to this flow");
+    }
+
+}
+
+
+
+/*
+
+
+ Packet stream -> flow reassembler -> flow1, flow2, flow3... -> for each flow, keep a sequence matcher for each sequence of cluster
+
+
+ */
\ No newline at end of file
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowReassembler.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowReassembler.java
new file mode 100644 (file)
index 0000000..e7b7304
--- /dev/null
@@ -0,0 +1,86 @@
+package edu.uci.iotproject.trafficreassembly.layer2;
+
+import edu.uci.iotproject.trafficreassembly.layer2.Layer2Flow;
+import edu.uci.iotproject.trafficreassembly.layer2.Layer2FlowReassemblerObserver;
+import org.pcap4j.core.PacketListener;
+import org.pcap4j.core.PcapPacket;
+import org.pcap4j.packet.EthernetPacket;
+import org.pcap4j.util.MacAddress;
+
+import java.util.*;
+
+/**
+ * Reassembles traffic flows at layer 2, i.e., for each combination of hosts, creates a list of packets exchanged
+ * between said hosts.
+ *
+ * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
+ * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
+ */
+public class Layer2FlowReassembler implements PacketListener {
+
+    /**
+     * Maps a pair of MAC addresses to the packets exchanged between the two hosts.
+     * The key is the concatenation of the two MAC addresses in hex string format, where the lexicographically smaller
+     * MAC is at the front of the string.
+     */
+    private final Map<String, Layer2Flow> mFlows = new HashMap<>();
+
+    private final List<Layer2FlowReassemblerObserver> mObservers = new ArrayList<>();
+
+    @Override
+    public void gotPacket(PcapPacket packet) {
+        // TODO: update to 802.11 packet...?
+        EthernetPacket ethPkt = packet.get(EthernetPacket.class);
+
+        MacAddress srcAddr = ethPkt.getHeader().getSrcAddr();
+        MacAddress dstAddr = ethPkt.getHeader().getDstAddr();
+
+        String key = keyFromAddresses(srcAddr, dstAddr);
+        // Create a new list if this pair of MAC addresses where not previously encountered and add packet to that list,
+        // or simply add to an existing list if one is present.
+        mFlows.computeIfAbsent(key, k -> {
+            Layer2Flow newFlow = new Layer2Flow(srcAddr, dstAddr);
+            // Inform observers of the new flow
+            mObservers.forEach(o -> o.onNewFlow(this, newFlow));
+            return newFlow;
+        }).addPacket(packet);
+    }
+
+    public void addObserver(Layer2FlowReassemblerObserver observer) {
+        mObservers.add(observer);
+    }
+
+    public void removeObserver(Layer2FlowReassemblerObserver observer) {
+        mObservers.remove(observer);
+    }
+
+    /**
+     * Get the traffic flow between two local endpoints ({@link MacAddress}es).
+     * @param addr1 The first endpoint.
+     * @param addr2 The second endpoint
+     * @return The traffic exchanged between the two endpoints.
+     */
+    public Layer2Flow getFlowForAddresses(MacAddress addr1, MacAddress addr2) {
+        return mFlows.get(keyFromAddresses(addr1, addr2));
+    }
+
+    /**
+     * Get all traffic flows, i.e., a traffic flow for each unique pair of endpoints (MAC addresses).
+     * @return All traffic flows.
+     */
+    public Collection<Layer2Flow> getFlows() {
+        return mFlows.values();
+    }
+
+    /**
+     * Given two {@link MacAddress}es, generates the corresponding key string used in {@link #mFlows}.
+     * @param addr1 The first address.
+     * @param addr2 The second address.
+     * @return the key string used in {@link #mFlows} corresponding to the two addresses.
+     */
+    private String keyFromAddresses(MacAddress addr1, MacAddress addr2) {
+        String addr1Str = addr1.toString();
+        String addr2Str = addr2.toString();
+        return addr1Str.compareTo(addr2Str) < 0 ? addr1Str + addr2Str : addr2Str + addr1Str;
+    }
+}
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowReassemblerObserver.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowReassemblerObserver.java
new file mode 100644 (file)
index 0000000..a71cd19
--- /dev/null
@@ -0,0 +1,19 @@
+package edu.uci.iotproject.trafficreassembly.layer2;
+
+/**
+ * For observing a {@link Layer2FlowReassembler}.
+ *
+ * @author Janus Varmarken
+ */
+public interface Layer2FlowReassemblerObserver {
+
+    /**
+     * Invoked when when a {@link Layer2FlowReassembler} detects a new flow (i.e., when it encounters traffic between two
+     * MAC addresses that has not previously communicated in the traffic trace).
+     *
+     * @param reassembler The {@link Layer2FlowReassembler} that detected the new flow.
+     * @param newFlow The new flow.
+     */
+    void onNewFlow(Layer2FlowReassembler reassembler, Layer2Flow newFlow);
+
+}
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/Conversation.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/Conversation.java
new file mode 100644 (file)
index 0000000..e89e81b
--- /dev/null
@@ -0,0 +1,585 @@
+package edu.uci.iotproject.trafficreassembly.layer3;
+
+import edu.uci.iotproject.analysis.TcpConversationUtils;
+import edu.uci.iotproject.util.PcapPacketUtils;
+import org.pcap4j.core.PcapPacket;
+import org.pcap4j.packet.IpV4Packet;
+import org.pcap4j.packet.Packet;
+import org.pcap4j.packet.TcpPacket;
+
+import java.util.*;
+
+/**
+ * Models a (TCP) conversation/connection/session/flow (packet's belonging to the same session between a client and a
+ * server).
+ * Holds a list of {@link PcapPacket}s identified as pertaining to the flow. Note that this list is <em>not</em>
+ * considered when determining equality of two {@code Conversation} instances in order to allow for a
+ * {@code Conversation} to function as a key in data structures such as {@link java.util.Map} and {@link java.util.Set}.
+ * See {@link #equals(Object)} for the definition of equality.
+ *
+ * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
+ * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
+ */
+public class Conversation {
+
+    /* Begin instance properties */
+    /**
+     * The IP of the host that is considered the client (i.e. the host that initiates the conversation)
+     * in this conversation.
+     */
+    private final String mClientIp;
+
+    /**
+     * The port number used by the host that is considered the client in this conversation.
+     */
+    private final int mClientPort;
+
+    /**
+     * The IP of the host that is considered the server (i.e. is the responder) in this conversation.
+     */
+    private final String mServerIp;
+
+    /**
+     * The port number used by the server in this conversation.
+     */
+    private final int mServerPort;
+
+    /**
+     * The list of packets (with payload) pertaining to this conversation.
+     */
+    private final List<PcapPacket> mPackets;
+
+    /**
+     * If {@link #isTls()} is {@code true}, this list contains the subset of {@link #mPackets} which are TLS Application
+     * Data packets.
+     */
+    private final List<PcapPacket> mTlsApplicationDataPackets;
+
+    /**
+     * Contains the sequence numbers used thus far by the host that is considered the <em>client</em> in this
+     * {@code Conversation}.
+     * Used for filtering out retransmissions.
+     */
+    private final Set<Integer> mSeqNumbersClient;
+
+    /**
+     * Contains the sequence numbers used thus far by the host that is considered the <em>server</em> in this
+     * {@code Conversation}.
+     * Used for filtering out retransmissions.
+     */
+    private final Set<Integer> mSeqNumbersSrv;
+
+    /**
+     * List of SYN packets pertaining to this conversation.
+     */
+    private final List<PcapPacket> mSynPackets;
+
+    /**
+     * List of pairs FINs and their corresponding ACKs associated with this conversation.
+     */
+    private final List<FinAckPair> mFinPackets;
+
+    /**
+     * List of RST packets associated with this conversation.
+     */
+    private final List<PcapPacket> mRstPackets;
+
+    /**
+     * Boolean to mark the packet as Application Data based on the previous packet that reaches MTU
+     */
+    private boolean mApplicationData;
+    /* End instance properties */
+
+    /**
+     * Factory method for creating a {@code Conversation} from a {@link PcapPacket}.
+     * @param pcapPacket The {@code PcapPacket} that wraps a TCP segment for which a {@code Conversation} is to be initiated.
+     * @param clientIsSrc If {@code true}, the source address and source port found in the IP datagram and TCP segment
+     *                    wrapped in the {@code PcapPacket} are regarded as pertaining to the client, and the destination
+     *                    address and destination port are regarded as pertaining to the server---and vice versa if set
+     *                    to {@code false}.
+     * @return A {@code Conversation} initiated with ip:port for client and server according to the direction of the packet.
+     */
+    public static Conversation fromPcapPacket(PcapPacket pcapPacket, boolean clientIsSrc) {
+        IpV4Packet ipPacket = pcapPacket.get(IpV4Packet.class);
+        TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class);
+        String clientIp = clientIsSrc ? ipPacket.getHeader().getSrcAddr().getHostAddress() :
+                ipPacket.getHeader().getDstAddr().getHostAddress();
+        String srvIp = clientIsSrc ? ipPacket.getHeader().getDstAddr().getHostAddress() :
+                ipPacket.getHeader().getSrcAddr().getHostAddress();
+        int clientPort = clientIsSrc ? tcpPacket.getHeader().getSrcPort().valueAsInt() :
+                tcpPacket.getHeader().getDstPort().valueAsInt();
+        int srvPort = clientIsSrc ? tcpPacket.getHeader().getDstPort().valueAsInt() :
+                tcpPacket.getHeader().getSrcPort().valueAsInt();
+        return new Conversation(clientIp, clientPort, srvIp, srvPort);
+    }
+
+    /**
+     * Constructs a new {@code Conversation}.
+     * @param clientIp The IP of the host that is considered the client (i.e. the host that initiates the conversation)
+     *                 in the conversation.
+     * @param clientPort The port number used by the client for the conversation.
+     * @param serverIp The IP of the host that is considered the server (i.e. is the responder) in the conversation.
+     * @param serverPort The port number used by the server for the conversation.
+     */
+    public Conversation(String clientIp, int clientPort, String serverIp, int serverPort) {
+        this.mClientIp = clientIp;
+        this.mClientPort = clientPort;
+        this.mServerIp = serverIp;
+        this.mServerPort = serverPort;
+        this.mPackets = new ArrayList<>();
+        this.mTlsApplicationDataPackets = new ArrayList<>();
+        this.mSeqNumbersClient = new HashSet<>();
+        this.mSeqNumbersSrv = new HashSet<>();
+        this.mSynPackets = new ArrayList<>();
+        this.mFinPackets = new ArrayList<>();
+        this.mRstPackets = new ArrayList<>();
+        this.mApplicationData = false;
+    }
+
+    /**
+     * Add a packet to the list of packets associated with this conversation.
+     * @param packet The packet that is to be added to (associated with) this conversation.
+     * @param ignoreRetransmissions Boolean value indicating if retransmissions should be ignored.
+     *                              If set to {@code true}, {@code packet} will <em>not</em> be added to the
+     *                              internal list of packets pertaining to this {@code Conversation}
+     *                              <em>iff</em> the sequence number of {@code packet} was already
+     *                              seen in a previous packet.
+     */
+    public void addPacket(PcapPacket packet, boolean ignoreRetransmissions) {
+        // Precondition: verify that packet does indeed pertain to conversation.
+        onAddPrecondition(packet);
+        if (ignoreRetransmissions && isRetransmission(packet)) {
+            // Packet is a retransmission. Ignore it.
+            return;
+        }
+        // Select direction-dependent set of sequence numbers seen so far and update it with sequence number of new packet.
+        addSeqNumber(packet);
+        // Finally add packet to list of packets pertaining to this conversation.
+        mPackets.add(packet);
+        // Preserve order of packets in list: sort according to timestamp.
+        if (mPackets.size() > 1 &&
+                mPackets.get(mPackets.size()-1).getTimestamp().isBefore(mPackets.get(mPackets.size()-2).getTimestamp())) {
+            Collections.sort(mPackets, (o1, o2) -> {
+                if (o1.getTimestamp().isBefore(o2.getTimestamp())) { return -1; }
+                else if (o2.getTimestamp().isBefore(o1.getTimestamp())) { return 1; }
+                else { return 0; }
+            });
+        }
+        // If TLS, inspect packet to see if it's a TLS Application Data packet, and if so add it to the list of TLS
+        // Application Data packets.
+        if (isTls()) {
+            TcpPacket tcpPacket = packet.get(TcpPacket.class);
+            Packet tcpPayload = tcpPacket.getPayload();
+            if (tcpPayload == null) {
+                return;
+            }
+            byte[] rawPayload = tcpPayload.getRawData();
+            // The SSL record header is at the front of the payload and is 5 bytes long.
+            // The SSL record header type field (the first byte) is set to 23 if it is an Application Data packet.
+            if (rawPayload != null && rawPayload.length >= 5) {
+                if (rawPayload[0] == 23) {
+                    mTlsApplicationDataPackets.add(packet);
+                    // Consider the following packet a data packet if this packet's size == MTU size 1448
+                    if (rawPayload.length >= 1448)
+                        mApplicationData = true;
+                } else if (rawPayload[0] == 20) {
+                    // Do nothing for now - CHANGE_CIPHER_SPEC
+                } else if (rawPayload[0] == 21) {
+                    // Do nothing for now - ALERT
+                } else if (rawPayload[0] == 22) {
+                    // Do nothing for now - HANDSHAKE
+                } else {
+                    // If it is TLS with payload, but rawPayload[0] != 23
+                    if (mApplicationData == true) {
+                        // It is a continuation of the previous packet if the previous packet reaches MTU size 1448 and
+                        // it is not either type 20, 21, or 22
+                        mTlsApplicationDataPackets.add(packet);
+                        if (rawPayload.length < 1448)
+                            mApplicationData = false;
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * Get a list of packets pertaining to this {@code Conversation}.
+     * The returned list is a read-only list.
+     * @return the list of packets pertaining to this {@code Conversation}.
+     */
+    public List<PcapPacket> getPackets() {
+        // Return read-only view to prevent external code from manipulating internal state (preserve invariant).
+        return Collections.unmodifiableList(mPackets);
+    }
+
+    /**
+     * Records a TCP SYN packet as pertaining to this conversation (adds it to the the internal list).
+     * Attempts to add duplicate SYN packets will be ignored, and the caller is made aware of the attempt to add a
+     * duplicate by the return value being {@code false}.
+     *
+     * @param synPacket A {@link PcapPacket} wrapping a TCP SYN packet.
+     * @return {@code true} if the packet was successfully added to this {@code Conversation}, {@code false} otherwise.
+     */
+    public boolean addSynPacket(PcapPacket synPacket) {
+        onAddPrecondition(synPacket);
+        final IpV4Packet synPacketIpSection = synPacket.get(IpV4Packet.class);
+        final TcpPacket synPacketTcpSection = synPacket.get(TcpPacket.class);
+        if (synPacketTcpSection == null || !synPacketTcpSection.getHeader().getSyn()) {
+            throw new IllegalArgumentException("Not a SYN packet.");
+        }
+        // We are only interested in recording one copy of the two SYN packets (one SYN packet in each direction), i.e.,
+        // we want to discard retransmitted SYN packets.
+        if (mSynPackets.size() >= 2) {
+            return false;
+        }
+        // Check the set of recorded SYN packets to see if we have already recorded a SYN packet going in the same
+        // direction as the packet given in the argument.
+        boolean matchingPrevSyn = mSynPackets.stream().anyMatch(p -> {
+            IpV4Packet pIp = p.get(IpV4Packet.class);
+            TcpPacket pTcp = p.get(TcpPacket.class);
+            boolean srcAddrMatch = synPacketIpSection.getHeader().getSrcAddr().getHostAddress().
+                    equals(pIp.getHeader().getSrcAddr().getHostAddress());
+            boolean dstAddrMatch = synPacketIpSection.getHeader().getDstAddr().getHostAddress().
+                    equals(pIp.getHeader().getDstAddr().getHostAddress());
+            boolean srcPortMatch = synPacketTcpSection.getHeader().getSrcPort().valueAsInt() ==
+                    pTcp.getHeader().getSrcPort().valueAsInt();
+            boolean dstPortMatch = synPacketTcpSection.getHeader().getDstPort().valueAsInt() ==
+                    pTcp.getHeader().getDstPort().valueAsInt();
+            return srcAddrMatch && dstAddrMatch && srcPortMatch && dstPortMatch;
+        });
+        if (matchingPrevSyn) {
+            return false;
+        }
+        // Update direction-dependent set of sequence numbers and record/log packet.
+        addSeqNumber(synPacket);
+        return mSynPackets.add(synPacket);
+
+        /*
+        mSynPackets.stream().anyMatch(p -> {
+            IpV4Packet pIp = p.get(IpV4Packet.class);
+            TcpPacket pTcp = p.get(TcpPacket.class);
+            boolean srcAddrMatch = synPacketIpSection.getHeader().getSrcAddr().getHostAddress().
+                    equals(pIp.getHeader().getSrcAddr().getHostAddress());
+            boolean dstAddrMatch = synPacketIpSection.getHeader().getDstAddr().getHostAddress().
+                    equals(pIp.getHeader().getDstAddr().getHostAddress());
+            boolean srcPortMatch = synPacketTcpSection.getHeader().getSrcPort().valueAsInt() ==
+                    pTcp.getHeader().getSrcPort().valueAsInt();
+            boolean dstPortMatch = synPacketTcpSection.getHeader().getDstPort().value() ==
+                    pTcp.getHeader().getDstPort().value();
+
+            boolean fourTupleMatch = srcAddrMatch && dstAddrMatch && srcPortMatch && dstPortMatch;
+
+            boolean seqNoMatch = synPacketTcpSection.getHeader().getSequenceNumber() ==
+                    pTcp.getHeader().getSequenceNumber();
+
+            if (fourTupleMatch && !seqNoMatch) {
+                // If the four tuple that identifies the conversation matches, but the sequence number is different,
+                // it means that this SYN packet is, in fact, an attempt to establish a **new** connection, and hence
+                // the given packet is NOT part of this conversation, even though the ip:port combinations are (by
+                // chance) selected such that they match this conversation.
+                throw new IllegalArgumentException("Attempt to add SYN packet that belongs to a different conversation " +
+                        "(which is identified by the same four tuple as this conversation)");
+            }
+            return fourTupleMatch && seqNoMatch;
+        });
+        */
+    }
+
+    /**
+     * Get a list of SYN packets pertaining to this {@code Conversation}.
+     * The returned list is a read-only list.
+     * @return the list of SYN packets pertaining to this {@code Conversation}.
+     */
+    public List<PcapPacket> getSynPackets() {
+        return Collections.unmodifiableList(mSynPackets);
+    }
+
+    /**
+     * Adds a TCP FIN packet to the list of TCP FIN packets associated with this conversation.
+     * @param finPacket The TCP FIN packet that is to be added to (associated with) this conversation.
+     */
+    public void addFinPacket(PcapPacket finPacket) {
+        // Precondition: verify that packet does indeed pertain to conversation.
+        onAddPrecondition(finPacket);
+        // TODO: should call addSeqNumber here?
+        addSeqNumber(finPacket);
+        mFinPackets.add(new FinAckPair(finPacket));
+    }
+
+    /**
+     * Attempt to ACK any FIN packets held by this conversation.
+     * @param ackPacket The ACK for a FIN previously added to this conversation.
+     */
+    public void attemptAcknowledgementOfFin(PcapPacket ackPacket) {
+        // Precondition: verify that the packet pertains to this conversation.
+        onAddPrecondition(ackPacket);
+        // Mark unack'ed FIN(s) that this ACK matches as ACK'ed (there might be more than one in case of retransmissions..?)
+        mFinPackets.replaceAll(finAckPair -> !finAckPair.isAcknowledged() && finAckPair.isCorrespondingAckPacket(ackPacket) ? new FinAckPair(finAckPair.getFinPacket(), ackPacket) : finAckPair);
+    }
+
+    /**
+     * Retrieves an unmodifiable view of the list of {@link FinAckPair}s associated with this {@code Conversation}.
+     * @return an unmodifiable view of the list of {@link FinAckPair}s associated with this {@code Conversation}.
+     */
+    public List<FinAckPair> getFinAckPairs() {
+        return Collections.unmodifiableList(mFinPackets);
+    }
+
+    /**
+     * Get if this {@code Conversation} is considered to have been gracefully shut down.
+     * A {@code Conversation} has been gracefully shut down if it contains a FIN+ACK pair for both directions
+     * (client to server, and server to client).
+     * @return {@code true} if the connection has been gracefully shut down, false otherwise.
+     */
+    public boolean isGracefullyShutdown() {
+        //  The conversation has been gracefully shut down if we have recorded a FIN from both the client and the server which have both been ack'ed.
+        return mFinPackets.stream().anyMatch(finAckPair -> finAckPair.isAcknowledged() && PcapPacketUtils.isSource(finAckPair.getFinPacket(), mClientIp, mClientPort)) &&
+                mFinPackets.stream().anyMatch(finAckPair -> finAckPair.isAcknowledged() && PcapPacketUtils.isSource(finAckPair.getFinPacket(), mServerIp, mServerPort));
+    }
+
+    /**
+     * Add a TCP segment for which the RST flag is set to this {@code Conversation}.
+     * @param packet A {@link PcapPacket} wrapping a TCP segment pertaining to this {@code Conversation} for which the
+     *               RST flag is set.
+     */
+    public void addRstPacket(PcapPacket packet) {
+        /*
+         * TODO:
+         * When now also keeping track of RST packets, should we also...?
+         * 1) Prevent later packets from being added once a RST segment has been added?
+         * 2) Extend 'isGracefullyShutdown()' to also consider RST segments, or add another method, 'isShutdown()' that
+         *    both considers FIN/ACK (graceful) as well as RST (abrupt/"ungraceful") shutdown?
+         * 3) Should it be impossible to associate more than one RST segment with each Conversation?
+         */
+        onAddPrecondition(packet);
+        TcpPacket tcpPacket = packet.get(TcpPacket.class);
+        if (tcpPacket == null || !tcpPacket.getHeader().getRst()) {
+            throw new IllegalArgumentException("not a RST packet");
+        }
+        mRstPackets.add(packet);
+    }
+
+    /**
+     * Get the TCP segments pertaining to this {@code Conversation} for which it was detected that the RST flag is set.
+     * @return the TCP segments pertaining to this {@code Conversation} for which it was detected that the RST flag is
+     *         set.
+     */
+    public List<PcapPacket> getRstPackets() {
+        return Collections.unmodifiableList(mRstPackets);
+    }
+
+    // =========================================================================================================
+    // We simply reuse equals and hashCode methods of String.class to be able to use this class as a key
+    // in a Map.
+
+    /**
+     * <em>Note:</em> currently, equality is determined based on pairwise equality of the elements of the four tuple
+     * ({@link #mClientIp}, {@link #mClientPort}, {@link #mServerIp}, {@link #mServerPort}) for {@code this} and
+     * {@code obj}.
+     * @param obj The object to test for equality with {@code this}.
+     * @return {@code true} if {@code obj} is considered equal to {@code this} based on the definition of equality given above.
+     */
+    @Override
+    public boolean equals(Object obj) {
+        return obj instanceof Conversation && this.toString().equals(obj.toString());
+    }
+
+    @Override
+    public int hashCode() {
+        return toString().hashCode();
+    }
+    // =========================================================================================================
+
+    @Override
+    public String toString() {
+        return String.format("%s:%d %s:%d", mClientIp, mClientPort, mServerIp, mServerPort);
+    }
+
+    /**
+     * Invoke to verify that the precondition holds when a caller attempts to add a packet to this {@code Conversation}.
+     * An {@link IllegalArgumentException} is thrown if the precondition is violated.
+     * @param packet the packet to be added to this {@code Conversation}
+     */
+    private void onAddPrecondition(PcapPacket packet) {
+        // Apply precondition to preserve class invariant: all packets in mPackets must match the 4 tuple that
+        // defines the conversation.
+        IpV4Packet ipPacket = Objects.requireNonNull(packet.get(IpV4Packet.class));
+        // For now we only support TCP flows.
+        TcpPacket tcpPacket = Objects.requireNonNull(packet.get(TcpPacket.class));
+        String ipSrc = ipPacket.getHeader().getSrcAddr().getHostAddress();
+        String ipDst = ipPacket.getHeader().getDstAddr().getHostAddress();
+        int srcPort = tcpPacket.getHeader().getSrcPort().valueAsInt();
+        int dstPort = tcpPacket.getHeader().getDstPort().valueAsInt();
+        String clientIp, serverIp;
+        int clientPort, serverPort;
+        if (ipSrc.equals(mClientIp)) {
+            clientIp = ipSrc;
+            clientPort = srcPort;
+            serverIp = ipDst;
+            serverPort = dstPort;
+        } else {
+            clientIp = ipDst;
+            clientPort = dstPort;
+            serverIp = ipSrc;
+            serverPort = srcPort;
+        }
+        if (!(clientIp.equals(mClientIp) && clientPort == mClientPort &&
+                serverIp.equals(mServerIp) && serverPort == mServerPort)) {
+            throw new IllegalArgumentException(
+                    String.format("Attempt to add packet that does not pertain to %s",
+                            Conversation.class.getSimpleName()));
+        }
+    }
+
+    /**
+     * <p>
+     *      Determines if the TCP packet contained in {@code packet} is a retransmission of a previously seen (logged)
+     *      packet.
+     * </p>
+     *
+     * <b>
+     *     TODO:
+     *     the current implementation, which uses a set of previously seen sequence numbers, will consider a segment
+     *     with a reused sequence number---occurring as a result of sequence number wrap around for a very long-lived
+     *     connection---as a retransmission (and may therefore end up discarding it even though it is in fact NOT a
+     *     retransmission). Ideas?
+     * </b>
+     *
+     * @param packet The packet.
+     * @return {@code true} if {@code packet} was determined to be a retransmission, {@code false} otherwise.
+     */
+    public boolean isRetransmission(PcapPacket packet) {
+        // Extract sequence number.
+        int seqNo = packet.get(TcpPacket.class).getHeader().getSequenceNumber();
+        switch (getDirection(packet)) {
+            case CLIENT_TO_SERVER:
+                return mSeqNumbersClient.contains(seqNo);
+            case SERVER_TO_CLIENT:
+                return mSeqNumbersSrv.contains(seqNo);
+            default:
+                throw new AssertionError(String.format("Unexpected value of enum '%s'",
+                        Direction.class.getSimpleName()));
+        }
+    }
+
+    /**
+     * <p>
+     *     Is this {@code Conversation} a TLS session?
+     * </p>
+     *
+     * <em>Note: the current implementation simply examines the port number(s) for 443; it does <b>not</b> verify if the
+     * application data is indeed encrypted.</em>
+     *
+     * @return {@code true} if this {@code Conversation} is interpreted as a TLS session, {@code false} otherwise.
+     */
+    public boolean isTls() {
+        /*
+         * TODO:
+         * - may want to change this to be "return mServerPort == 443 || mClientPort == 443;" in order to also detect
+         *   TLS in those cases where it is not possible to correctly label who is the client and who is the server,
+         *   i.e., when the trace does not contain the SYN/SYNACK exchange.
+         * - current implementation relies on the server using the conventional TLS port number; may instead want to
+         *   inspect the first 4 bytes of each potential TLS packet to see if they match the SSL record header.
+         *
+         * 08/31/18: Added unconvetional TLS ports used by WeMo plugs and LiFX bulb.
+         * 09/20/18: Moved hardcoded ports to other class to allow other classes to query the set of TLS ports.
+         */
+        return TcpConversationUtils.isTlsPort(mServerPort);
+    }
+
+    /**
+     * If this {@code Conversation} is backing a TLS session (i.e., if the value of {@link #isTls()} is {@code true}),
+     * get the packets labeled as TLS Application Data packets. This is a subset of the full set of payload-carrying
+     * packets (as returned by {@link #getPackets()}). An exception is thrown if this method is invoked on a
+     * {@code Conversation} for which {@link #isTls()} returns {@code false}.
+     *
+     * @return A list containing exactly those packets that could be identified as TLS Application Data packets (through
+     *         inspecting of the SSL record header). The list may be empty, if no TLS application data packets have been
+     *         recorded for this {@code Conversation}.
+     */
+    public List<PcapPacket> getTlsApplicationDataPackets() {
+        if (!isTls()) {
+            throw new NoSuchElementException("cannot get TLS Application Data packets for non-TLS TCP conversation");
+        }
+        return Collections.unmodifiableList(mTlsApplicationDataPackets);
+    }
+
+    /**
+     * Extracts the TCP sequence number from {@code packet} and adds it to the proper set of sequence numbers by
+     * analyzing the direction of the packet.
+     * @param packet A TCP packet (wrapped in a {@code PcapPacket}) that was added to this conversation and whose
+     *               sequence number is to be recorded as seen.
+     */
+    private void addSeqNumber(PcapPacket packet) {
+        // Note: below check is redundant if client code is correct as the call to check the precondition should already
+        // have been made by the addXPacket method that invokes this method. As such, the call below may be removed in
+        // favor of speed, but the improvement will be minor, hence the added safety may be worth it.
+        onAddPrecondition(packet);
+        // Extract sequence number.
+        int seqNo = packet.get(TcpPacket.class).getHeader().getSequenceNumber();
+        // Determine direction of packet and add packet's sequence number to corresponding set of sequence numbers.
+        switch (getDirection(packet)) {
+            case CLIENT_TO_SERVER:
+                // Client to server packet.
+                mSeqNumbersClient.add(seqNo);
+                break;
+            case SERVER_TO_CLIENT:
+                // Server to client packet.
+                mSeqNumbersSrv.add(seqNo);
+                break;
+            default:
+                throw new AssertionError(String.format("Unexpected value of enum '%s'",
+                        Direction.class.getSimpleName()));
+        }
+    }
+
+    /**
+     * Determine the direction of {@code packet}. An {@link IllegalArgumentException} is thrown if {@code packet} does
+     * not pertain to this conversation.
+     *
+     * @param packet The packet whose direction is to be determined.
+     * @return A {@link Direction} indicating the direction of the packet.
+     */
+    public Direction getDirection(PcapPacket packet) {
+        IpV4Packet ipPacket = packet.get(IpV4Packet.class);
+        String ipSrc = ipPacket.getHeader().getSrcAddr().getHostAddress();
+        String ipDst = ipPacket.getHeader().getDstAddr().getHostAddress();
+        // Determine direction of packet.
+        if (ipSrc.equals(mClientIp) && ipDst.equals(mServerIp)) {
+            // Client to server packet.
+            return Direction.CLIENT_TO_SERVER;
+        } else if (ipSrc.equals(mServerIp) && ipDst.equals(mClientIp)) {
+            // Server to client packet.
+            return Direction.SERVER_TO_CLIENT;
+        } else {
+            throw new IllegalArgumentException("getDirection: packet not related to " + getClass().getSimpleName());
+        }
+    }
+
+    /**
+     * Utility enum for expressing the direction of a packet pertaining to this {@code Conversation}.
+     */
+    public enum Direction {
+
+        CLIENT_TO_SERVER {
+            @Override
+            public String toCompactString() {
+                return "*";
+            }
+        },
+        SERVER_TO_CLIENT {
+            @Override
+            public String toCompactString() {
+                return "";
+            }
+        };
+
+        /**
+         * Get a compact string representation of this {@code Direction}.
+         * @return a compact string representation of this {@code Direction}.
+         */
+        abstract public String toCompactString();
+
+    }
+
+}
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/FinAckPair.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/FinAckPair.java
new file mode 100644 (file)
index 0000000..fe4f032
--- /dev/null
@@ -0,0 +1,134 @@
+package edu.uci.iotproject.trafficreassembly.layer3;
+
+import org.pcap4j.core.PcapPacket;
+import org.pcap4j.packet.IpV4Packet;
+import org.pcap4j.packet.TcpPacket;
+
+/**
+ * Groups a FIN packet and its corresponding ACK packet. <b>Immutable and thread safe</b>.
+ *
+ * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
+ * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
+ */
+public class FinAckPair {
+
+    private final PcapPacket mFinPacket;
+    private final PcapPacket mCorrespondingAckPacket;
+
+    /**
+     * Constructs a {@code FinAckPair} given a FIN packet.
+     * The corresponding ACK packet field is set to {@code null}.
+     * @param finPacket A FIN packet.
+     */
+    public FinAckPair(PcapPacket finPacket) {
+        if (!finPacket.get(TcpPacket.class).getHeader().getFin()) {
+            throw new IllegalArgumentException("not a FIN packet");
+        }
+        mFinPacket = finPacket;
+        mCorrespondingAckPacket = null;
+    }
+
+    /**
+     * Constructs a {@code FinAckPair} given a FIN and an ACK packet.
+     * @param finPacket A FIN packet.
+     * @param correspondingAckPacket The ACK packet corresponding to {@code finPacket}.
+     */
+    public FinAckPair(PcapPacket finPacket, PcapPacket correspondingAckPacket) {
+        // Enforce class invariant, i.e. that the FIN and ACK are related.
+        // Note that it is indirectly checked whether finPacket is indeed a FIN packet
+        // as isCorrespondingAckPacket calls the single parameter constructor.
+        if (!FinAckPair.isCorrespondingAckPacket(finPacket, correspondingAckPacket)) {
+            throw new IllegalArgumentException("FIN and ACK not related");
+        }
+        mFinPacket = finPacket;
+        mCorrespondingAckPacket = correspondingAckPacket;
+    }
+
+    /**
+     * Get the FIN packet of this pair.
+     * @return the FIN packet of this pair.
+     */
+    public PcapPacket getFinPacket() {
+        return mFinPacket;
+    }
+
+    /**
+     * Get the corresponding ACK packet of this pair, if any.
+     * @return the corresponding ACK packet of this pair, if any.
+     */
+    public PcapPacket getCorrespondingAckPacket() {
+        return mCorrespondingAckPacket;
+    }
+
+    /**
+     * Was the FIN in this {@code FinAckPair} acknowledged?
+     *
+     * @return {@code true} if the corresponding ACK has been set in this {@code FinAckPair}.
+     */
+    public boolean isAcknowledged() {
+        return mFinPacket != null && mCorrespondingAckPacket != null;
+    }
+
+    /**
+     * Checks if a given packet is an ACK corresponding to the FIN packet in this {@code FinAckPair}.
+     * @return {@code true} if {@code packet} is an ACK that corresponds to the FIN in this pair, {@code false} otherwise.
+     */
+    public boolean isCorrespondingAckPacket(PcapPacket packet) {
+        IpV4Packet inputIpPacket = packet.get(IpV4Packet.class);
+        TcpPacket inputTcpPacket = packet.get(TcpPacket.class);
+        if (inputIpPacket == null || inputTcpPacket == null || !inputTcpPacket.getHeader().getAck()) {
+            return false;
+        }
+
+        IpV4Packet finIpPacket = mFinPacket.get(IpV4Packet.class);
+        TcpPacket finTcpPacket = mFinPacket.get(TcpPacket.class);
+
+        // Extract (srcIp:port,dstIp:port) for input and member (FIN) packets.
+        String inputPacketIpSrc = inputIpPacket.getHeader().getSrcAddr().getHostAddress();
+        String inputPacketIpDst = inputIpPacket.getHeader().getDstAddr().getHostAddress();
+        int inputPacketPortSrc = inputTcpPacket.getHeader().getSrcPort().valueAsInt();
+        int inputPacketPortDst = inputTcpPacket.getHeader().getDstPort().valueAsInt();
+        String finPacketIpSrc = finIpPacket.getHeader().getSrcAddr().getHostAddress();
+        String finPacketIpDst = finIpPacket.getHeader().getDstAddr().getHostAddress();
+        int finPacketPortSrc = finTcpPacket.getHeader().getSrcPort().valueAsInt();
+        int finPacketPortDst = finTcpPacket.getHeader().getDstPort().valueAsInt();
+
+        // For the two packets to be related, the dst of one must be the src of the other.
+        // Split into multiple if statements for readability. First check IP fields, then ports.
+        if (!(inputPacketIpDst.equals(finPacketIpSrc) && finPacketIpDst.equals(inputPacketIpSrc))) {
+            return false;
+        }
+        if (!(inputPacketPortDst == finPacketPortSrc && finPacketPortDst == inputPacketPortSrc)) {
+            return false;
+        }
+
+        // Packets are (most likely) related (part of same conversation/stream).
+        // Now all that is left for us to check is if the sequence numbers match.
+        // Note: recall that the FIN packet advances the seq numbers by 1,
+        // so the ACK number will be one larger than the seq. number in the FIN packet.
+        return inputTcpPacket.getHeader().getAcknowledgmentNumber() == finTcpPacket.getHeader().getSequenceNumber() + 1;
+    }
+
+    /**
+     * Static method to check if two given packets are a FIN and the corresponding ACK packet.
+     * The purpose of this method is a workaround to enforce the class invariant in the two parameter constructor.
+     * Specifically, the following should be avoided:
+     * <pre>
+     *     public FinAckPair(PcapPacket finPacket, PcapPacket correspondingAckPacket) {
+     *         mFinPacket = finPacket;
+     *         // Below line is considered bad practice as the object has not been fully initialized at this stage.
+     *         if (!this.isCorrespondingAckPacket(correspondingAckPacket)) {
+     *             // ... throw exception
+     *         }
+     *     }
+     * </pre>
+     * @param finPacket The FIN packet.
+     * @param ackPacket The ACK packet that is to be checked if it corresponds to the given FIN packet.
+     * @return {@code true} if the ACK corresponds to the FIN, {@code false} otherwise.
+     */
+    private static boolean isCorrespondingAckPacket(PcapPacket finPacket, PcapPacket ackPacket) {
+        FinAckPair tmp = new FinAckPair(finPacket);
+        return tmp.isCorrespondingAckPacket(ackPacket);
+    }
+
+}
diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/TcpReassembler.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/TcpReassembler.java
new file mode 100644 (file)
index 0000000..e150875
--- /dev/null
@@ -0,0 +1,260 @@
+package edu.uci.iotproject.trafficreassembly.layer3;
+
+import org.pcap4j.core.PacketListener;
+import org.pcap4j.core.PcapPacket;
+import org.pcap4j.packet.*;
+
+import java.util.*;
+
+/**
+ * Reassembles TCP conversations (streams).
+ * <b>Note: current version only supports TCP over IPv4.</b>
+ *
+ * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
+ * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
+ */
+public class TcpReassembler implements PacketListener {
+
+    /**
+     * Holds <em>open</em> {@link Conversation}s, i.e., {@code Conversation}s that have <em>not</em> been detected as
+     * (gracefully) terminated based on the set of packets observed thus far.
+     * A {@link Conversation} is moved to {@link #mTerminatedConversations} if it can be determined that it is has
+     * terminated. Termination can be detected by a) observing two {@link FinAckPair}s, one in each direction, (graceful
+     * termination, see {@link Conversation#isGracefullyShutdown()}) or b) by observing a SYN packet that matches the
+     * four tuple of an existing {@code Conversation}, but which holds a <em>different</em> sequence number than the
+     * same-direction SYN packet recorded for the {@code Conversation}.
+     * <p>
+     * Note that due to limitations of the {@link Set} interface (specifically, there is no {@code get(T t)} method),
+     * we have to resort to a {@link Map} (in which keys map to themselves) to "mimic" a set with {@code get(T t)}
+     * functionality.
+     *
+     * @see <a href="https://stackoverflow.com/questions/7283338/getting-an-element-from-a-set">this question on StackOverflow.com</a>
+     */
+    private final Map<Conversation, Conversation> mOpenConversations = new HashMap<>();
+
+    /**
+     * Holds <em>terminated</em> {@link Conversation}s.
+     */
+    private final List<Conversation> mTerminatedConversations = new ArrayList<>();
+
+    @Override
+    public void gotPacket(PcapPacket pcapPacket) {
+        IpV4Packet ipPacket = pcapPacket.get(IpV4Packet.class);
+        TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class);
+
+        if (ipPacket == null || tcpPacket == null) {
+            return;
+        }
+        // ... TODO?
+        processPacket(pcapPacket);
+//        Class clazz = pcapPacket.getClass();
+//        RadiotapPacket radiotapPacket = pcapPacket.get(RadiotapPacket.class);
+//        Dot11ManagementPacket dot11ManagementPacket = pcapPacket.get(Dot11ManagementPacket.class);
+//        if (dot11ManagementPacket != null) {
+//            return;
+//        }
+//        if (radiotapPacket != null) {
+//            processRadiotapPacket(pcapPacket);
+//        }
+    }
+
+    /**
+     * Get the reassembled TCP connections. Note that if this is called while packets are still being processed (by
+     * calls to {@link #gotPacket(PcapPacket)}), the behavior is undefined and the returned list may be inconsistent.
+     * @return The reassembled TCP connections.
+     */
+    public List<Conversation> getTcpConversations() {
+        ArrayList<Conversation> combined = new ArrayList<>();
+        combined.addAll(mTerminatedConversations);
+        combined.addAll(mOpenConversations.values());
+        return combined;
+    }
+
+    private void processRadiotapPacket(PcapPacket pcapPacket) {
+        RadiotapPacket radiotapPacket = pcapPacket.get(RadiotapPacket.class);
+
+        RadiotapPacket.RadiotapHeader header = radiotapPacket.getHeader();
+        short length = header.getLength();
+        ArrayList<RadiotapPacket.RadiotapData> radiotapData = header.getDataFields();
+        // TODO: We can handle this 802.11 QoS data by creating our own class
+        // TODO: We only need to handle the first few bytes for source, destination, receiver, and transmitter
+        // TODO: addresses
+        Packet dataPacket = radiotapPacket.getPayload();
+        int dataLength = dataPacket.length();
+    }
+
+    private void processPacket(PcapPacket pcapPacket) {
+        TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class);
+        // Handle client connection initiation attempts.
+        if (tcpPacket.getHeader().getSyn() && !tcpPacket.getHeader().getAck()) {
+            // A segment with the SYN flag set, but no ACK flag indicates that a client is attempting to initiate a new
+            // connection.
+            processNewConnectionRequest(pcapPacket);
+            return;
+        }
+        // Handle server connection initiation acknowledgement
+        if (tcpPacket.getHeader().getSyn() && tcpPacket.getHeader().getAck()) {
+            // A segment with both the SYN and ACK flags set indicates that the server has accepted the client's request
+            // to initiate a new connection.
+            processNewConnectionAck(pcapPacket);
+            return;
+        }
+        // Handle resets
+        if (tcpPacket.getHeader().getRst()) {
+            processRstPacket(pcapPacket);
+            return;
+        }
+        // Handle FINs
+        if (tcpPacket.getHeader().getFin()) {
+            // Handle FIN packet.
+            processFinPacket(pcapPacket);
+        }
+        // Handle ACKs (currently only ACKs of FINS)
+        if (tcpPacket.getHeader().getAck()) {
+            processAck(pcapPacket);
+        }
+        // Handle packets that carry payload (application data).
+        if (tcpPacket.getPayload() != null) {
+            processPayloadPacket(pcapPacket);
+        }
+    }
+
+    private void processNewConnectionRequest(PcapPacket clientSynPacket) {
+        // A SYN w/o ACK always originates from the client.
+        Conversation conv = Conversation.fromPcapPacket(clientSynPacket, true);
+        conv.addSynPacket(clientSynPacket);
+        // Is there an ongoing conversation for the same four tuple (clientIp, clientPort, serverIp, serverPort) as
+        // found in the new SYN packet?
+        Conversation ongoingConv = mOpenConversations.get(conv);
+        if (ongoingConv != null) {
+            if (ongoingConv.isRetransmission(clientSynPacket)) {
+                // SYN retransmission detected, do nothing.
+                return;
+                // TODO: the way retransmission detection is implemented may cause a bug for connections where we have
+                // not recorded the initial SYN, but only the SYN ACK, as retransmission is determined by comparing the
+                // sequence numbers of initial SYNs -- and if no initial SYN is present for the Conversation, the new
+                // SYN will be interpreted as a retransmission. Possible fix: let isRentransmission ALWAYS return false
+                // when presented with a SYN packet when the Conversation already holds a SYN ACK packet?
+            } else {
+                // New SYN has different sequence number than SYN recorded for ongoingConv, so this must be an attempt
+                // to establish a new conversation with the same four tuple as ongoingConv.
+                // Mark existing connection as terminated.
+                // TODO: is this 100% theoretically correct, e.g., if many connection attempts are made back to back? And RST packets?
+                mTerminatedConversations.add(ongoingConv);
+                mOpenConversations.remove(ongoingConv);
+            }
+        }
+        // Finally, update the map of open connections with the new connection.
+        mOpenConversations.put(conv, conv);
+    }
+
+
+    /*
+     * TODO a problem across the board for all processXPacket methods below:
+     * if we start the capture in the middle of a TCP connection, we will not have an entry for the conversation in the
+     * map as we have not seen the initial SYN packet.
+     * Two ways we can address this:
+     * a) Perform null-checks and ignore packets for which we have not seen SYN
+     *    + easy to get correct
+     *    - we discard data (issue for long-lived connections!)
+     * b) Add a corresponding conversation entry whenever we encounter a packet that does not map to a conversation
+     *    + we consider all data
+     *    - not immediately clear if this will introduce bugs (incorrectly mapping packets to wrong conversations?)
+     *
+     *  [[[ I went with option b) for now; see getOngoingConversationOrCreateNew(PcapPacket pcapPacket). ]]]
+     */
+
+    private void processNewConnectionAck(PcapPacket srvSynPacket) {
+        // Find the corresponding ongoing connection, if any (if we start the capture just *after* the initial SYN, no
+        // ongoing conversation entry will exist, so it must be created in that case).
+//        Conversation conv = mOpenConversations.get(Conversation.fromPcapPacket(srvSynPacket, false));
+        Conversation conv = getOngoingConversationOrCreateNew(srvSynPacket);
+        // Note: exploits &&'s short-circuit operation: only attempts to add non-retransmissions.
+        if (!conv.isRetransmission(srvSynPacket) && !conv.addSynPacket(srvSynPacket)) {
+            // For safety/debugging: if NOT a retransmission and add fails,
+            // something has gone terribly wrong/invariant is broken.
+//            throw new AssertionError("Attempt to add SYN ACK packet that was NOT a retransmission failed." +
+//                    Conversation.class.getSimpleName() + " invariant broken.");
+        }
+    }
+
+    private void processRstPacket(PcapPacket rstPacket) {
+        Conversation conv = getOngoingConversationOrCreateNew(rstPacket);
+        // Add RST packet to conversation.
+        conv.addRstPacket(rstPacket);
+        // Move conversation to set of terminated conversations.
+        mTerminatedConversations.add(conv);
+        mOpenConversations.remove(conv, conv);
+    }
+
+    private void processFinPacket(PcapPacket finPacket) {
+//        getOngoingConversationForPacket(finPacket).addFinPacket(finPacket);
+        getOngoingConversationOrCreateNew(finPacket).addFinPacket(finPacket);
+    }
+
+    private void processAck(PcapPacket ackPacket) {
+//        getOngoingConversationForPacket(ackPacket).attemptAcknowledgementOfFin(ackPacket);
+        // Note that unlike the style for SYN, FIN, and payload packets, for "ACK only" packets, we want to avoid
+        // creating a new conversation.
+        Conversation conv = getOngoingConversationForPacket(ackPacket);
+        if (conv != null) {
+            // The ACK may be an ACK of a FIN, so attempt to mark the FIN as ack'ed.
+            conv.attemptAcknowledgementOfFin(ackPacket);
+            if (conv.isGracefullyShutdown()) {
+                // Move conversation to set of terminated conversations.
+                mTerminatedConversations.add(conv);
+                mOpenConversations.remove(conv);
+            }
+        }
+        // Note: add (additional) processing of ACKs (that are not ACKs of FINs) as necessary here...
+    }
+
+    private void processPayloadPacket(PcapPacket pcapPacket) {
+//        getOngoingConversationForPacket(pcapPacket).addPacket(pcapPacket, true);
+        getOngoingConversationOrCreateNew(pcapPacket).addPacket(pcapPacket, true);
+    }
+
+    /**
+     * Locates an ongoing conversation (if any) that {@code pcapPacket} pertains to.
+     * @param pcapPacket The packet that is to be mapped to an ongoing {@code Conversation}.
+     * @return The {@code Conversation} matching {@code pcapPacket} or {@code null} if there is no match.
+     */
+    private Conversation getOngoingConversationForPacket(PcapPacket pcapPacket) {
+        // We cannot know if this is a client-to-server or server-to-client packet without trying both options...
+        Conversation conv = mOpenConversations.get(Conversation.fromPcapPacket(pcapPacket, true));
+        if (conv == null) {
+            conv = mOpenConversations.get(Conversation.fromPcapPacket(pcapPacket, false));
+        }
+        return conv;
+    }
+
+    /**
+     * Like {@link #getOngoingConversationForPacket(PcapPacket)}, but creates and inserts a new {@code Conversation}
+     * into {@link #mOpenConversations} if no open conversation is found (i.e., in the case that
+     * {@link #getOngoingConversationForPacket(PcapPacket)} returns {@code null}).
+     *
+     * @param pcapPacket The packet that is to be mapped to an ongoing {@code Conversation}.
+     * @return The existing, ongoing {@code Conversation} matching {@code pcapPacket} or the newly created one in case
+     *         no match was found.
+     */
+    private Conversation getOngoingConversationOrCreateNew(PcapPacket pcapPacket) {
+        Conversation conv = getOngoingConversationForPacket(pcapPacket);
+        if (conv == null) {
+            TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class);
+            if (tcpPacket.getHeader().getSyn() && tcpPacket.getHeader().getAck()) {
+                // A SYN ACK packet always originates from the server (it is a reply to the initial SYN packet from the client)
+                conv = Conversation.fromPcapPacket(pcapPacket, false);
+            } else {
+                // TODO: can we do anything else but arbitrarily select who is designated as the server in this case?
+                // We can check if the IP prefix matches a local IP when handling traffic observed inside the local
+                // network, but that obviously won't be a useful strategy for an observer at the WAN port.
+                String srcIp = pcapPacket.get(IpV4Packet.class).getHeader().getSrcAddr().getHostAddress();
+                // TODO: REPLACE THE ROUTER'S IP WITH A PARAMETER!!!
+                boolean clientIsSrc = srcIp.startsWith("10.") || srcIp.startsWith("192.168.") || srcIp.equals("128.195.205.105");
+                conv = Conversation.fromPcapPacket(pcapPacket, clientIsSrc);
+            }
+            mOpenConversations.put(conv, conv);
+        }
+        return conv;
+    }
+}
index cc958ee..f07072a 100644 (file)
@@ -1,6 +1,6 @@
 package edu.uci.iotproject.util;
 
-import edu.uci.iotproject.Conversation;
+import edu.uci.iotproject.trafficreassembly.layer3.Conversation;
 import edu.uci.iotproject.analysis.PcapPacketPair;
 import edu.uci.iotproject.analysis.TcpConversationUtils;
 import edu.uci.iotproject.analysis.TriggerTrafficExtractor;