From: Janus Varmarken Date: Sun, 13 Jan 2019 04:18:53 +0000 (-0800) Subject: Reorganize code by creating a package for code that reassembles traffic flows at... X-Git-Url: http://plrg.eecs.uci.edu/git/?a=commitdiff_plain;h=a315a7187514946ab1a40b973dae2e9f6b2dfad3;p=pingpong.git Reorganize code by creating a package for code that reassembles traffic flows at different layers --- diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Conversation.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Conversation.java deleted file mode 100644 index 490dc4f..0000000 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Conversation.java +++ /dev/null @@ -1,585 +0,0 @@ -package edu.uci.iotproject; - -import edu.uci.iotproject.analysis.TcpConversationUtils; -import edu.uci.iotproject.util.PcapPacketUtils; -import org.pcap4j.core.PcapPacket; -import org.pcap4j.packet.IpV4Packet; -import org.pcap4j.packet.Packet; -import org.pcap4j.packet.TcpPacket; - -import java.util.*; - -/** - * Models a (TCP) conversation/connection/session/flow (packet's belonging to the same session between a client and a - * server). - * Holds a list of {@link PcapPacket}s identified as pertaining to the flow. Note that this list is not - * considered when determining equality of two {@code Conversation} instances in order to allow for a - * {@code Conversation} to function as a key in data structures such as {@link java.util.Map} and {@link java.util.Set}. - * See {@link #equals(Object)} for the definition of equality. - * - * @author Janus Varmarken {@literal } - * @author Rahmadi Trimananda {@literal } - */ -public class Conversation { - - /* Begin instance properties */ - /** - * The IP of the host that is considered the client (i.e. the host that initiates the conversation) - * in this conversation. - */ - private final String mClientIp; - - /** - * The port number used by the host that is considered the client in this conversation. - */ - private final int mClientPort; - - /** - * The IP of the host that is considered the server (i.e. is the responder) in this conversation. - */ - private final String mServerIp; - - /** - * The port number used by the server in this conversation. - */ - private final int mServerPort; - - /** - * The list of packets (with payload) pertaining to this conversation. - */ - private final List mPackets; - - /** - * If {@link #isTls()} is {@code true}, this list contains the subset of {@link #mPackets} which are TLS Application - * Data packets. - */ - private final List mTlsApplicationDataPackets; - - /** - * Contains the sequence numbers used thus far by the host that is considered the client in this - * {@code Conversation}. - * Used for filtering out retransmissions. - */ - private final Set mSeqNumbersClient; - - /** - * Contains the sequence numbers used thus far by the host that is considered the server in this - * {@code Conversation}. - * Used for filtering out retransmissions. - */ - private final Set mSeqNumbersSrv; - - /** - * List of SYN packets pertaining to this conversation. - */ - private final List mSynPackets; - - /** - * List of pairs FINs and their corresponding ACKs associated with this conversation. - */ - private final List mFinPackets; - - /** - * List of RST packets associated with this conversation. - */ - private final List mRstPackets; - - /** - * Boolean to mark the packet as Application Data based on the previous packet that reaches MTU - */ - private boolean mApplicationData; - /* End instance properties */ - - /** - * Factory method for creating a {@code Conversation} from a {@link PcapPacket}. - * @param pcapPacket The {@code PcapPacket} that wraps a TCP segment for which a {@code Conversation} is to be initiated. - * @param clientIsSrc If {@code true}, the source address and source port found in the IP datagram and TCP segment - * wrapped in the {@code PcapPacket} are regarded as pertaining to the client, and the destination - * address and destination port are regarded as pertaining to the server---and vice versa if set - * to {@code false}. - * @return A {@code Conversation} initiated with ip:port for client and server according to the direction of the packet. - */ - public static Conversation fromPcapPacket(PcapPacket pcapPacket, boolean clientIsSrc) { - IpV4Packet ipPacket = pcapPacket.get(IpV4Packet.class); - TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class); - String clientIp = clientIsSrc ? ipPacket.getHeader().getSrcAddr().getHostAddress() : - ipPacket.getHeader().getDstAddr().getHostAddress(); - String srvIp = clientIsSrc ? ipPacket.getHeader().getDstAddr().getHostAddress() : - ipPacket.getHeader().getSrcAddr().getHostAddress(); - int clientPort = clientIsSrc ? tcpPacket.getHeader().getSrcPort().valueAsInt() : - tcpPacket.getHeader().getDstPort().valueAsInt(); - int srvPort = clientIsSrc ? tcpPacket.getHeader().getDstPort().valueAsInt() : - tcpPacket.getHeader().getSrcPort().valueAsInt(); - return new Conversation(clientIp, clientPort, srvIp, srvPort); - } - - /** - * Constructs a new {@code Conversation}. - * @param clientIp The IP of the host that is considered the client (i.e. the host that initiates the conversation) - * in the conversation. - * @param clientPort The port number used by the client for the conversation. - * @param serverIp The IP of the host that is considered the server (i.e. is the responder) in the conversation. - * @param serverPort The port number used by the server for the conversation. - */ - public Conversation(String clientIp, int clientPort, String serverIp, int serverPort) { - this.mClientIp = clientIp; - this.mClientPort = clientPort; - this.mServerIp = serverIp; - this.mServerPort = serverPort; - this.mPackets = new ArrayList<>(); - this.mTlsApplicationDataPackets = new ArrayList<>(); - this.mSeqNumbersClient = new HashSet<>(); - this.mSeqNumbersSrv = new HashSet<>(); - this.mSynPackets = new ArrayList<>(); - this.mFinPackets = new ArrayList<>(); - this.mRstPackets = new ArrayList<>(); - this.mApplicationData = false; - } - - /** - * Add a packet to the list of packets associated with this conversation. - * @param packet The packet that is to be added to (associated with) this conversation. - * @param ignoreRetransmissions Boolean value indicating if retransmissions should be ignored. - * If set to {@code true}, {@code packet} will not be added to the - * internal list of packets pertaining to this {@code Conversation} - * iff the sequence number of {@code packet} was already - * seen in a previous packet. - */ - public void addPacket(PcapPacket packet, boolean ignoreRetransmissions) { - // Precondition: verify that packet does indeed pertain to conversation. - onAddPrecondition(packet); - if (ignoreRetransmissions && isRetransmission(packet)) { - // Packet is a retransmission. Ignore it. - return; - } - // Select direction-dependent set of sequence numbers seen so far and update it with sequence number of new packet. - addSeqNumber(packet); - // Finally add packet to list of packets pertaining to this conversation. - mPackets.add(packet); - // Preserve order of packets in list: sort according to timestamp. - if (mPackets.size() > 1 && - mPackets.get(mPackets.size()-1).getTimestamp().isBefore(mPackets.get(mPackets.size()-2).getTimestamp())) { - Collections.sort(mPackets, (o1, o2) -> { - if (o1.getTimestamp().isBefore(o2.getTimestamp())) { return -1; } - else if (o2.getTimestamp().isBefore(o1.getTimestamp())) { return 1; } - else { return 0; } - }); - } - // If TLS, inspect packet to see if it's a TLS Application Data packet, and if so add it to the list of TLS - // Application Data packets. - if (isTls()) { - TcpPacket tcpPacket = packet.get(TcpPacket.class); - Packet tcpPayload = tcpPacket.getPayload(); - if (tcpPayload == null) { - return; - } - byte[] rawPayload = tcpPayload.getRawData(); - // The SSL record header is at the front of the payload and is 5 bytes long. - // The SSL record header type field (the first byte) is set to 23 if it is an Application Data packet. - if (rawPayload != null && rawPayload.length >= 5) { - if (rawPayload[0] == 23) { - mTlsApplicationDataPackets.add(packet); - // Consider the following packet a data packet if this packet's size == MTU size 1448 - if (rawPayload.length >= 1448) - mApplicationData = true; - } else if (rawPayload[0] == 20) { - // Do nothing for now - CHANGE_CIPHER_SPEC - } else if (rawPayload[0] == 21) { - // Do nothing for now - ALERT - } else if (rawPayload[0] == 22) { - // Do nothing for now - HANDSHAKE - } else { - // If it is TLS with payload, but rawPayload[0] != 23 - if (mApplicationData == true) { - // It is a continuation of the previous packet if the previous packet reaches MTU size 1448 and - // it is not either type 20, 21, or 22 - mTlsApplicationDataPackets.add(packet); - if (rawPayload.length < 1448) - mApplicationData = false; - } - } - } - } - } - - /** - * Get a list of packets pertaining to this {@code Conversation}. - * The returned list is a read-only list. - * @return the list of packets pertaining to this {@code Conversation}. - */ - public List getPackets() { - // Return read-only view to prevent external code from manipulating internal state (preserve invariant). - return Collections.unmodifiableList(mPackets); - } - - /** - * Records a TCP SYN packet as pertaining to this conversation (adds it to the the internal list). - * Attempts to add duplicate SYN packets will be ignored, and the caller is made aware of the attempt to add a - * duplicate by the return value being {@code false}. - * - * @param synPacket A {@link PcapPacket} wrapping a TCP SYN packet. - * @return {@code true} if the packet was successfully added to this {@code Conversation}, {@code false} otherwise. - */ - public boolean addSynPacket(PcapPacket synPacket) { - onAddPrecondition(synPacket); - final IpV4Packet synPacketIpSection = synPacket.get(IpV4Packet.class); - final TcpPacket synPacketTcpSection = synPacket.get(TcpPacket.class); - if (synPacketTcpSection == null || !synPacketTcpSection.getHeader().getSyn()) { - throw new IllegalArgumentException("Not a SYN packet."); - } - // We are only interested in recording one copy of the two SYN packets (one SYN packet in each direction), i.e., - // we want to discard retransmitted SYN packets. - if (mSynPackets.size() >= 2) { - return false; - } - // Check the set of recorded SYN packets to see if we have already recorded a SYN packet going in the same - // direction as the packet given in the argument. - boolean matchingPrevSyn = mSynPackets.stream().anyMatch(p -> { - IpV4Packet pIp = p.get(IpV4Packet.class); - TcpPacket pTcp = p.get(TcpPacket.class); - boolean srcAddrMatch = synPacketIpSection.getHeader().getSrcAddr().getHostAddress(). - equals(pIp.getHeader().getSrcAddr().getHostAddress()); - boolean dstAddrMatch = synPacketIpSection.getHeader().getDstAddr().getHostAddress(). - equals(pIp.getHeader().getDstAddr().getHostAddress()); - boolean srcPortMatch = synPacketTcpSection.getHeader().getSrcPort().valueAsInt() == - pTcp.getHeader().getSrcPort().valueAsInt(); - boolean dstPortMatch = synPacketTcpSection.getHeader().getDstPort().valueAsInt() == - pTcp.getHeader().getDstPort().valueAsInt(); - return srcAddrMatch && dstAddrMatch && srcPortMatch && dstPortMatch; - }); - if (matchingPrevSyn) { - return false; - } - // Update direction-dependent set of sequence numbers and record/log packet. - addSeqNumber(synPacket); - return mSynPackets.add(synPacket); - - /* - mSynPackets.stream().anyMatch(p -> { - IpV4Packet pIp = p.get(IpV4Packet.class); - TcpPacket pTcp = p.get(TcpPacket.class); - boolean srcAddrMatch = synPacketIpSection.getHeader().getSrcAddr().getHostAddress(). - equals(pIp.getHeader().getSrcAddr().getHostAddress()); - boolean dstAddrMatch = synPacketIpSection.getHeader().getDstAddr().getHostAddress(). - equals(pIp.getHeader().getDstAddr().getHostAddress()); - boolean srcPortMatch = synPacketTcpSection.getHeader().getSrcPort().valueAsInt() == - pTcp.getHeader().getSrcPort().valueAsInt(); - boolean dstPortMatch = synPacketTcpSection.getHeader().getDstPort().value() == - pTcp.getHeader().getDstPort().value(); - - boolean fourTupleMatch = srcAddrMatch && dstAddrMatch && srcPortMatch && dstPortMatch; - - boolean seqNoMatch = synPacketTcpSection.getHeader().getSequenceNumber() == - pTcp.getHeader().getSequenceNumber(); - - if (fourTupleMatch && !seqNoMatch) { - // If the four tuple that identifies the conversation matches, but the sequence number is different, - // it means that this SYN packet is, in fact, an attempt to establish a **new** connection, and hence - // the given packet is NOT part of this conversation, even though the ip:port combinations are (by - // chance) selected such that they match this conversation. - throw new IllegalArgumentException("Attempt to add SYN packet that belongs to a different conversation " + - "(which is identified by the same four tuple as this conversation)"); - } - return fourTupleMatch && seqNoMatch; - }); - */ - } - - /** - * Get a list of SYN packets pertaining to this {@code Conversation}. - * The returned list is a read-only list. - * @return the list of SYN packets pertaining to this {@code Conversation}. - */ - public List getSynPackets() { - return Collections.unmodifiableList(mSynPackets); - } - - /** - * Adds a TCP FIN packet to the list of TCP FIN packets associated with this conversation. - * @param finPacket The TCP FIN packet that is to be added to (associated with) this conversation. - */ - public void addFinPacket(PcapPacket finPacket) { - // Precondition: verify that packet does indeed pertain to conversation. - onAddPrecondition(finPacket); - // TODO: should call addSeqNumber here? - addSeqNumber(finPacket); - mFinPackets.add(new FinAckPair(finPacket)); - } - - /** - * Attempt to ACK any FIN packets held by this conversation. - * @param ackPacket The ACK for a FIN previously added to this conversation. - */ - public void attemptAcknowledgementOfFin(PcapPacket ackPacket) { - // Precondition: verify that the packet pertains to this conversation. - onAddPrecondition(ackPacket); - // Mark unack'ed FIN(s) that this ACK matches as ACK'ed (there might be more than one in case of retransmissions..?) - mFinPackets.replaceAll(finAckPair -> !finAckPair.isAcknowledged() && finAckPair.isCorrespondingAckPacket(ackPacket) ? new FinAckPair(finAckPair.getFinPacket(), ackPacket) : finAckPair); - } - - /** - * Retrieves an unmodifiable view of the list of {@link FinAckPair}s associated with this {@code Conversation}. - * @return an unmodifiable view of the list of {@link FinAckPair}s associated with this {@code Conversation}. - */ - public List getFinAckPairs() { - return Collections.unmodifiableList(mFinPackets); - } - - /** - * Get if this {@code Conversation} is considered to have been gracefully shut down. - * A {@code Conversation} has been gracefully shut down if it contains a FIN+ACK pair for both directions - * (client to server, and server to client). - * @return {@code true} if the connection has been gracefully shut down, false otherwise. - */ - public boolean isGracefullyShutdown() { - // The conversation has been gracefully shut down if we have recorded a FIN from both the client and the server which have both been ack'ed. - return mFinPackets.stream().anyMatch(finAckPair -> finAckPair.isAcknowledged() && PcapPacketUtils.isSource(finAckPair.getFinPacket(), mClientIp, mClientPort)) && - mFinPackets.stream().anyMatch(finAckPair -> finAckPair.isAcknowledged() && PcapPacketUtils.isSource(finAckPair.getFinPacket(), mServerIp, mServerPort)); - } - - /** - * Add a TCP segment for which the RST flag is set to this {@code Conversation}. - * @param packet A {@link PcapPacket} wrapping a TCP segment pertaining to this {@code Conversation} for which the - * RST flag is set. - */ - public void addRstPacket(PcapPacket packet) { - /* - * TODO: - * When now also keeping track of RST packets, should we also...? - * 1) Prevent later packets from being added once a RST segment has been added? - * 2) Extend 'isGracefullyShutdown()' to also consider RST segments, or add another method, 'isShutdown()' that - * both considers FIN/ACK (graceful) as well as RST (abrupt/"ungraceful") shutdown? - * 3) Should it be impossible to associate more than one RST segment with each Conversation? - */ - onAddPrecondition(packet); - TcpPacket tcpPacket = packet.get(TcpPacket.class); - if (tcpPacket == null || !tcpPacket.getHeader().getRst()) { - throw new IllegalArgumentException("not a RST packet"); - } - mRstPackets.add(packet); - } - - /** - * Get the TCP segments pertaining to this {@code Conversation} for which it was detected that the RST flag is set. - * @return the TCP segments pertaining to this {@code Conversation} for which it was detected that the RST flag is - * set. - */ - public List getRstPackets() { - return Collections.unmodifiableList(mRstPackets); - } - - // ========================================================================================================= - // We simply reuse equals and hashCode methods of String.class to be able to use this class as a key - // in a Map. - - /** - * Note: currently, equality is determined based on pairwise equality of the elements of the four tuple - * ({@link #mClientIp}, {@link #mClientPort}, {@link #mServerIp}, {@link #mServerPort}) for {@code this} and - * {@code obj}. - * @param obj The object to test for equality with {@code this}. - * @return {@code true} if {@code obj} is considered equal to {@code this} based on the definition of equality given above. - */ - @Override - public boolean equals(Object obj) { - return obj instanceof Conversation && this.toString().equals(obj.toString()); - } - - @Override - public int hashCode() { - return toString().hashCode(); - } - // ========================================================================================================= - - @Override - public String toString() { - return String.format("%s:%d %s:%d", mClientIp, mClientPort, mServerIp, mServerPort); - } - - /** - * Invoke to verify that the precondition holds when a caller attempts to add a packet to this {@code Conversation}. - * An {@link IllegalArgumentException} is thrown if the precondition is violated. - * @param packet the packet to be added to this {@code Conversation} - */ - private void onAddPrecondition(PcapPacket packet) { - // Apply precondition to preserve class invariant: all packets in mPackets must match the 4 tuple that - // defines the conversation. - IpV4Packet ipPacket = Objects.requireNonNull(packet.get(IpV4Packet.class)); - // For now we only support TCP flows. - TcpPacket tcpPacket = Objects.requireNonNull(packet.get(TcpPacket.class)); - String ipSrc = ipPacket.getHeader().getSrcAddr().getHostAddress(); - String ipDst = ipPacket.getHeader().getDstAddr().getHostAddress(); - int srcPort = tcpPacket.getHeader().getSrcPort().valueAsInt(); - int dstPort = tcpPacket.getHeader().getDstPort().valueAsInt(); - String clientIp, serverIp; - int clientPort, serverPort; - if (ipSrc.equals(mClientIp)) { - clientIp = ipSrc; - clientPort = srcPort; - serverIp = ipDst; - serverPort = dstPort; - } else { - clientIp = ipDst; - clientPort = dstPort; - serverIp = ipSrc; - serverPort = srcPort; - } - if (!(clientIp.equals(mClientIp) && clientPort == mClientPort && - serverIp.equals(mServerIp) && serverPort == mServerPort)) { - throw new IllegalArgumentException( - String.format("Attempt to add packet that does not pertain to %s", - Conversation.class.getSimpleName())); - } - } - - /** - *

- * Determines if the TCP packet contained in {@code packet} is a retransmission of a previously seen (logged) - * packet. - *

- * - * - * TODO: - * the current implementation, which uses a set of previously seen sequence numbers, will consider a segment - * with a reused sequence number---occurring as a result of sequence number wrap around for a very long-lived - * connection---as a retransmission (and may therefore end up discarding it even though it is in fact NOT a - * retransmission). Ideas? - * - * - * @param packet The packet. - * @return {@code true} if {@code packet} was determined to be a retransmission, {@code false} otherwise. - */ - public boolean isRetransmission(PcapPacket packet) { - // Extract sequence number. - int seqNo = packet.get(TcpPacket.class).getHeader().getSequenceNumber(); - switch (getDirection(packet)) { - case CLIENT_TO_SERVER: - return mSeqNumbersClient.contains(seqNo); - case SERVER_TO_CLIENT: - return mSeqNumbersSrv.contains(seqNo); - default: - throw new AssertionError(String.format("Unexpected value of enum '%s'", - Direction.class.getSimpleName())); - } - } - - /** - *

- * Is this {@code Conversation} a TLS session? - *

- * - * Note: the current implementation simply examines the port number(s) for 443; it does not verify if the - * application data is indeed encrypted. - * - * @return {@code true} if this {@code Conversation} is interpreted as a TLS session, {@code false} otherwise. - */ - public boolean isTls() { - /* - * TODO: - * - may want to change this to be "return mServerPort == 443 || mClientPort == 443;" in order to also detect - * TLS in those cases where it is not possible to correctly label who is the client and who is the server, - * i.e., when the trace does not contain the SYN/SYNACK exchange. - * - current implementation relies on the server using the conventional TLS port number; may instead want to - * inspect the first 4 bytes of each potential TLS packet to see if they match the SSL record header. - * - * 08/31/18: Added unconvetional TLS ports used by WeMo plugs and LiFX bulb. - * 09/20/18: Moved hardcoded ports to other class to allow other classes to query the set of TLS ports. - */ - return TcpConversationUtils.isTlsPort(mServerPort); - } - - /** - * If this {@code Conversation} is backing a TLS session (i.e., if the value of {@link #isTls()} is {@code true}), - * get the packets labeled as TLS Application Data packets. This is a subset of the full set of payload-carrying - * packets (as returned by {@link #getPackets()}). An exception is thrown if this method is invoked on a - * {@code Conversation} for which {@link #isTls()} returns {@code false}. - * - * @return A list containing exactly those packets that could be identified as TLS Application Data packets (through - * inspecting of the SSL record header). The list may be empty, if no TLS application data packets have been - * recorded for this {@code Conversation}. - */ - public List getTlsApplicationDataPackets() { - if (!isTls()) { - throw new NoSuchElementException("cannot get TLS Application Data packets for non-TLS TCP conversation"); - } - return Collections.unmodifiableList(mTlsApplicationDataPackets); - } - - /** - * Extracts the TCP sequence number from {@code packet} and adds it to the proper set of sequence numbers by - * analyzing the direction of the packet. - * @param packet A TCP packet (wrapped in a {@code PcapPacket}) that was added to this conversation and whose - * sequence number is to be recorded as seen. - */ - private void addSeqNumber(PcapPacket packet) { - // Note: below check is redundant if client code is correct as the call to check the precondition should already - // have been made by the addXPacket method that invokes this method. As such, the call below may be removed in - // favor of speed, but the improvement will be minor, hence the added safety may be worth it. - onAddPrecondition(packet); - // Extract sequence number. - int seqNo = packet.get(TcpPacket.class).getHeader().getSequenceNumber(); - // Determine direction of packet and add packet's sequence number to corresponding set of sequence numbers. - switch (getDirection(packet)) { - case CLIENT_TO_SERVER: - // Client to server packet. - mSeqNumbersClient.add(seqNo); - break; - case SERVER_TO_CLIENT: - // Server to client packet. - mSeqNumbersSrv.add(seqNo); - break; - default: - throw new AssertionError(String.format("Unexpected value of enum '%s'", - Direction.class.getSimpleName())); - } - } - - /** - * Determine the direction of {@code packet}. An {@link IllegalArgumentException} is thrown if {@code packet} does - * not pertain to this conversation. - * - * @param packet The packet whose direction is to be determined. - * @return A {@link Direction} indicating the direction of the packet. - */ - public Direction getDirection(PcapPacket packet) { - IpV4Packet ipPacket = packet.get(IpV4Packet.class); - String ipSrc = ipPacket.getHeader().getSrcAddr().getHostAddress(); - String ipDst = ipPacket.getHeader().getDstAddr().getHostAddress(); - // Determine direction of packet. - if (ipSrc.equals(mClientIp) && ipDst.equals(mServerIp)) { - // Client to server packet. - return Direction.CLIENT_TO_SERVER; - } else if (ipSrc.equals(mServerIp) && ipDst.equals(mClientIp)) { - // Server to client packet. - return Direction.SERVER_TO_CLIENT; - } else { - throw new IllegalArgumentException("getDirection: packet not related to " + getClass().getSimpleName()); - } - } - - /** - * Utility enum for expressing the direction of a packet pertaining to this {@code Conversation}. - */ - public enum Direction { - - CLIENT_TO_SERVER { - @Override - public String toCompactString() { - return "*"; - } - }, - SERVER_TO_CLIENT { - @Override - public String toCompactString() { - return ""; - } - }; - - /** - * Get a compact string representation of this {@code Direction}. - * @return a compact string representation of this {@code Direction}. - */ - abstract public String toCompactString(); - - } - -} diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FinAckPair.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FinAckPair.java deleted file mode 100644 index d4451f3..0000000 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FinAckPair.java +++ /dev/null @@ -1,134 +0,0 @@ -package edu.uci.iotproject; - -import org.pcap4j.core.PcapPacket; -import org.pcap4j.packet.IpV4Packet; -import org.pcap4j.packet.TcpPacket; - -/** - * Groups a FIN packet and its corresponding ACK packet. Immutable and thread safe. - * - * @author Janus Varmarken {@literal } - * @author Rahmadi Trimananda {@literal } - */ -public class FinAckPair { - - private final PcapPacket mFinPacket; - private final PcapPacket mCorrespondingAckPacket; - - /** - * Constructs a {@code FinAckPair} given a FIN packet. - * The corresponding ACK packet field is set to {@code null}. - * @param finPacket A FIN packet. - */ - public FinAckPair(PcapPacket finPacket) { - if (!finPacket.get(TcpPacket.class).getHeader().getFin()) { - throw new IllegalArgumentException("not a FIN packet"); - } - mFinPacket = finPacket; - mCorrespondingAckPacket = null; - } - - /** - * Constructs a {@code FinAckPair} given a FIN and an ACK packet. - * @param finPacket A FIN packet. - * @param correspondingAckPacket The ACK packet corresponding to {@code finPacket}. - */ - public FinAckPair(PcapPacket finPacket, PcapPacket correspondingAckPacket) { - // Enforce class invariant, i.e. that the FIN and ACK are related. - // Note that it is indirectly checked whether finPacket is indeed a FIN packet - // as isCorrespondingAckPacket calls the single parameter constructor. - if (!FinAckPair.isCorrespondingAckPacket(finPacket, correspondingAckPacket)) { - throw new IllegalArgumentException("FIN and ACK not related"); - } - mFinPacket = finPacket; - mCorrespondingAckPacket = correspondingAckPacket; - } - - /** - * Get the FIN packet of this pair. - * @return the FIN packet of this pair. - */ - public PcapPacket getFinPacket() { - return mFinPacket; - } - - /** - * Get the corresponding ACK packet of this pair, if any. - * @return the corresponding ACK packet of this pair, if any. - */ - public PcapPacket getCorrespondingAckPacket() { - return mCorrespondingAckPacket; - } - - /** - * Was the FIN in this {@code FinAckPair} acknowledged? - * - * @return {@code true} if the corresponding ACK has been set in this {@code FinAckPair}. - */ - public boolean isAcknowledged() { - return mFinPacket != null && mCorrespondingAckPacket != null; - } - - /** - * Checks if a given packet is an ACK corresponding to the FIN packet in this {@code FinAckPair}. - * @return {@code true} if {@code packet} is an ACK that corresponds to the FIN in this pair, {@code false} otherwise. - */ - public boolean isCorrespondingAckPacket(PcapPacket packet) { - IpV4Packet inputIpPacket = packet.get(IpV4Packet.class); - TcpPacket inputTcpPacket = packet.get(TcpPacket.class); - if (inputIpPacket == null || inputTcpPacket == null || !inputTcpPacket.getHeader().getAck()) { - return false; - } - - IpV4Packet finIpPacket = mFinPacket.get(IpV4Packet.class); - TcpPacket finTcpPacket = mFinPacket.get(TcpPacket.class); - - // Extract (srcIp:port,dstIp:port) for input and member (FIN) packets. - String inputPacketIpSrc = inputIpPacket.getHeader().getSrcAddr().getHostAddress(); - String inputPacketIpDst = inputIpPacket.getHeader().getDstAddr().getHostAddress(); - int inputPacketPortSrc = inputTcpPacket.getHeader().getSrcPort().valueAsInt(); - int inputPacketPortDst = inputTcpPacket.getHeader().getDstPort().valueAsInt(); - String finPacketIpSrc = finIpPacket.getHeader().getSrcAddr().getHostAddress(); - String finPacketIpDst = finIpPacket.getHeader().getDstAddr().getHostAddress(); - int finPacketPortSrc = finTcpPacket.getHeader().getSrcPort().valueAsInt(); - int finPacketPortDst = finTcpPacket.getHeader().getDstPort().valueAsInt(); - - // For the two packets to be related, the dst of one must be the src of the other. - // Split into multiple if statements for readability. First check IP fields, then ports. - if (!(inputPacketIpDst.equals(finPacketIpSrc) && finPacketIpDst.equals(inputPacketIpSrc))) { - return false; - } - if (!(inputPacketPortDst == finPacketPortSrc && finPacketPortDst == inputPacketPortSrc)) { - return false; - } - - // Packets are (most likely) related (part of same conversation/stream). - // Now all that is left for us to check is if the sequence numbers match. - // Note: recall that the FIN packet advances the seq numbers by 1, - // so the ACK number will be one larger than the seq. number in the FIN packet. - return inputTcpPacket.getHeader().getAcknowledgmentNumber() == finTcpPacket.getHeader().getSequenceNumber() + 1; - } - - /** - * Static method to check if two given packets are a FIN and the corresponding ACK packet. - * The purpose of this method is a workaround to enforce the class invariant in the two parameter constructor. - * Specifically, the following should be avoided: - * 
-     *     public FinAckPair(PcapPacket finPacket, PcapPacket correspondingAckPacket) {
-     *         mFinPacket = finPacket;
-     *         // Below line is considered bad practice as the object has not been fully initialized at this stage.
-     *         if (!this.isCorrespondingAckPacket(correspondingAckPacket)) {
-     *             // ... throw exception
-     *         }
-     *     }
-     *
- * @param finPacket The FIN packet. - * @param ackPacket The ACK packet that is to be checked if it corresponds to the given FIN packet. - * @return {@code true} if the ACK corresponds to the FIN, {@code false} otherwise. - */ - private static boolean isCorrespondingAckPacket(PcapPacket finPacket, PcapPacket ackPacket) { - FinAckPair tmp = new FinAckPair(finPacket); - return tmp.isCorrespondingAckPacket(ackPacket); - } - -} diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FlowPatternFinder.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FlowPatternFinder.java index 1d0a9ba..c384852 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FlowPatternFinder.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FlowPatternFinder.java @@ -3,6 +3,7 @@ package edu.uci.iotproject; import edu.uci.iotproject.comparison.ComparisonFunctions; import edu.uci.iotproject.comparison.CompleteMatchPatternComparisonResult; import edu.uci.iotproject.comparison.PatternComparisonTask; +import edu.uci.iotproject.trafficreassembly.layer3.Conversation; import org.pcap4j.core.NotOpenException; import org.pcap4j.core.PcapHandle; import org.pcap4j.core.PcapNativeException; @@ -15,7 +16,6 @@ import java.io.*; import java.net.UnknownHostException; import java.text.DateFormat; import java.text.SimpleDateFormat; -import java.time.temporal.ChronoField; import java.util.*; import java.util.concurrent.*; diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/L2FlowReassembler.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/L2FlowReassembler.java deleted file mode 100644 index b09189f..0000000 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/L2FlowReassembler.java +++ /dev/null @@ -1,84 +0,0 @@ -package edu.uci.iotproject; - -import org.pcap4j.core.PacketListener; -import org.pcap4j.core.PcapPacket; -import org.pcap4j.packet.EthernetPacket; -import org.pcap4j.util.MacAddress; - -import java.util.*; - -/** - * Reassembles traffic flows at layer 2, i.e., for each combination of hosts, creates a list of packets exchanged - * between said hosts. - * - * @author Janus Varmarken {@literal } - * @author Rahmadi Trimananda {@literal } - */ -public class L2FlowReassembler implements PacketListener { - - /** - * Maps a pair of MAC addresses to the packets exchanged between the two hosts. - * The key is the concatenation of the two MAC addresses in hex string format, where the lexicographically smaller - * MAC is at the front of the string. - */ - private final Map mFlows = new HashMap<>(); - - private final List mObservers = new ArrayList<>(); - - @Override - public void gotPacket(PcapPacket packet) { - // TODO: update to 802.11 packet...? - EthernetPacket ethPkt = packet.get(EthernetPacket.class); - - MacAddress srcAddr = ethPkt.getHeader().getSrcAddr(); - MacAddress dstAddr = ethPkt.getHeader().getDstAddr(); - - String key = keyFromAddresses(srcAddr, dstAddr); - // Create a new list if this pair of MAC addresses where not previously encountered and add packet to that list, - // or simply add to an existing list if one is present. - mFlows.computeIfAbsent(key, k -> { - Layer2Flow newFlow = new Layer2Flow(srcAddr, dstAddr); - // Inform observers of the new flow - mObservers.forEach(o -> o.onNewFlow(this, newFlow)); - return newFlow; - }).addPacket(packet); - } - - public void addObserver(Layer2FlowReassemblerObserver observer) { - mObservers.add(observer); - } - - public void removeObserver(Layer2FlowReassemblerObserver observer) { - mObservers.remove(observer); - } - - /** - * Get the traffic flow between two local endpoints ({@link MacAddress}es). - * @param addr1 The first endpoint. - * @param addr2 The second endpoint - * @return The traffic exchanged between the two endpoints. - */ - public Layer2Flow getFlowForAddresses(MacAddress addr1, MacAddress addr2) { - return mFlows.get(keyFromAddresses(addr1, addr2)); - } - - /** - * Get all traffic flows, i.e., a traffic flow for each unique pair of endpoints (MAC addresses). - * @return All traffic flows. - */ - public Collection getFlows() { - return mFlows.values(); - } - - /** - * Given two {@link MacAddress}es, generates the corresponding key string used in {@link #mFlows}. - * @param addr1 The first address. - * @param addr2 The second address. - * @return the key string used in {@link #mFlows} corresponding to the two addresses. - */ - private String keyFromAddresses(MacAddress addr1, MacAddress addr2) { - String addr1Str = addr1.toString(); - String addr2Str = addr2.toString(); - return addr1Str.compareTo(addr2Str) < 0 ? addr1Str + addr2Str : addr2Str + addr1Str; - } -} diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Layer2Flow.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Layer2Flow.java deleted file mode 100644 index d50a911..0000000 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Layer2Flow.java +++ /dev/null @@ -1,79 +0,0 @@ -package edu.uci.iotproject; - -import edu.uci.iotproject.detection.Layer2FlowObserver; -import org.pcap4j.core.PcapPacket; -import org.pcap4j.packet.EthernetPacket; -import org.pcap4j.util.MacAddress; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -/** - * The packets exchanged between two endpoints (MAC addresses). - * - * @author Janus Varmarken - */ -public class Layer2Flow { - - private final MacAddress mEndpoint1; - private final MacAddress mEndpoint2; - - private final List mFlowObservers = new ArrayList<>(); - - public Layer2Flow(MacAddress endpoint1, MacAddress endpoint2) { - mEndpoint1 = endpoint1; - mEndpoint2 = endpoint2; - } - - public void addFlowObserver(Layer2FlowObserver observer) { - mFlowObservers.add(observer); - } - - public void removeFlowObserver(Layer2FlowObserver observer) { - mFlowObservers.remove(observer); - } - - /** - * The packets in the flow. - */ - private final List mPackets = new ArrayList<>(); - - /** - * Add a packet to this flow. - * @param packet The packet that is to be added to the flow. - */ - public void addPacket(PcapPacket packet) { - verifyAddresses(packet); - mPackets.add(packet); - // Notify flow observers of the new packet - mFlowObservers.forEach(o -> o.onNewPacket(this, packet)); - } - - public List getPackets() { - return Collections.unmodifiableList(mPackets); - } - - private void verifyAddresses(PcapPacket packet) { - EthernetPacket ethPkt = packet.get(EthernetPacket.class); - MacAddress srcAddr = ethPkt.getHeader().getSrcAddr(); - MacAddress dstAddr = ethPkt.getHeader().getDstAddr(); - if ((mEndpoint1.equals(srcAddr) && mEndpoint2.equals(dstAddr)) || - (mEndpoint1.equals(dstAddr) && mEndpoint2.equals(srcAddr))) { - // All is good. - return; - } - throw new IllegalArgumentException("Mismatch in MACs: packet does not pertain to this flow"); - } - -} - - - -/* - - - Packet stream -> flow reassembler -> flow1, flow2, flow3... -> for each flow, keep a sequence matcher for each sequence of cluster - - - */ \ No newline at end of file diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Layer2FlowReassemblerObserver.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Layer2FlowReassemblerObserver.java deleted file mode 100644 index c9eb2f7..0000000 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Layer2FlowReassemblerObserver.java +++ /dev/null @@ -1,19 +0,0 @@ -package edu.uci.iotproject; - -/** - * For observing a {@link L2FlowReassembler}. - * - * @author Janus Varmarken - */ -public interface Layer2FlowReassemblerObserver { - - /** - * Invoked when when a {@link L2FlowReassembler} detects a new flow (i.e., when it encounters traffic between two - * MAC addresses that has not previously communicated in the traffic trace). - * - * @param reassembler The {@link L2FlowReassembler} that detected the new flow. - * @param newFlow The new flow. - */ - void onNewFlow(L2FlowReassembler reassembler, Layer2Flow newFlow); - -} diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java index 5b9a54e..a7e9a6c 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java @@ -3,10 +3,9 @@ package edu.uci.iotproject; import static edu.uci.iotproject.analysis.UserAction.Type; import edu.uci.iotproject.analysis.*; -import edu.uci.iotproject.comparison.seqalignment.ExtractedSequence; -import edu.uci.iotproject.comparison.seqalignment.SequenceAlignment; -import edu.uci.iotproject.comparison.seqalignment.SequenceExtraction; import edu.uci.iotproject.io.TriggerTimesFileReader; +import edu.uci.iotproject.trafficreassembly.layer3.Conversation; +import edu.uci.iotproject.trafficreassembly.layer3.TcpReassembler; import edu.uci.iotproject.util.PcapPacketUtils; import edu.uci.iotproject.util.PrintUtils; import org.apache.commons.math3.stat.clustering.Cluster; @@ -15,8 +14,6 @@ import org.pcap4j.core.*; import org.pcap4j.packet.namednumber.DataLinkType; import java.io.EOFException; -import java.io.File; -import java.io.PrintWriter; import java.net.UnknownHostException; import java.time.Instant; import java.util.*; diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/TcpReassembler.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/TcpReassembler.java deleted file mode 100644 index 72f0c1c..0000000 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/TcpReassembler.java +++ /dev/null @@ -1,260 +0,0 @@ -package edu.uci.iotproject; - -import org.pcap4j.core.PacketListener; -import org.pcap4j.core.PcapPacket; -import org.pcap4j.packet.*; - -import java.util.*; - -/** - * Reassembles TCP conversations (streams). - * Note: current version only supports TCP over IPv4. - * - * @author Janus Varmarken {@literal } - * @author Rahmadi Trimananda {@literal } - */ -public class TcpReassembler implements PacketListener { - - /** - * Holds open {@link Conversation}s, i.e., {@code Conversation}s that have not been detected as - * (gracefully) terminated based on the set of packets observed thus far. - * A {@link Conversation} is moved to {@link #mTerminatedConversations} if it can be determined that it is has - * terminated. Termination can be detected by a) observing two {@link FinAckPair}s, one in each direction, (graceful - * termination, see {@link Conversation#isGracefullyShutdown()}) or b) by observing a SYN packet that matches the - * four tuple of an existing {@code Conversation}, but which holds a different sequence number than the - * same-direction SYN packet recorded for the {@code Conversation}. - *

- * Note that due to limitations of the {@link Set} interface (specifically, there is no {@code get(T t)} method), - * we have to resort to a {@link Map} (in which keys map to themselves) to "mimic" a set with {@code get(T t)} - * functionality. - * - * @see this question on StackOverflow.com - */ - private final Map mOpenConversations = new HashMap<>(); - - /** - * Holds terminated {@link Conversation}s. - */ - private final List mTerminatedConversations = new ArrayList<>(); - - @Override - public void gotPacket(PcapPacket pcapPacket) { - IpV4Packet ipPacket = pcapPacket.get(IpV4Packet.class); - TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class); - - if (ipPacket == null || tcpPacket == null) { - return; - } - // ... TODO? - processPacket(pcapPacket); -// Class clazz = pcapPacket.getClass(); -// RadiotapPacket radiotapPacket = pcapPacket.get(RadiotapPacket.class); -// Dot11ManagementPacket dot11ManagementPacket = pcapPacket.get(Dot11ManagementPacket.class); -// if (dot11ManagementPacket != null) { -// return; -// } -// if (radiotapPacket != null) { -// processRadiotapPacket(pcapPacket); -// } - } - - /** - * Get the reassembled TCP connections. Note that if this is called while packets are still being processed (by - * calls to {@link #gotPacket(PcapPacket)}), the behavior is undefined and the returned list may be inconsistent. - * @return The reassembled TCP connections. - */ - public List getTcpConversations() { - ArrayList combined = new ArrayList<>(); - combined.addAll(mTerminatedConversations); - combined.addAll(mOpenConversations.values()); - return combined; - } - - private void processRadiotapPacket(PcapPacket pcapPacket) { - RadiotapPacket radiotapPacket = pcapPacket.get(RadiotapPacket.class); - - RadiotapPacket.RadiotapHeader header = radiotapPacket.getHeader(); - short length = header.getLength(); - ArrayList radiotapData = header.getDataFields(); - // TODO: We can handle this 802.11 QoS data by creating our own class - // TODO: We only need to handle the first few bytes for source, destination, receiver, and transmitter - // TODO: addresses - Packet dataPacket = radiotapPacket.getPayload(); - int dataLength = dataPacket.length(); - } - - private void processPacket(PcapPacket pcapPacket) { - TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class); - // Handle client connection initiation attempts. - if (tcpPacket.getHeader().getSyn() && !tcpPacket.getHeader().getAck()) { - // A segment with the SYN flag set, but no ACK flag indicates that a client is attempting to initiate a new - // connection. - processNewConnectionRequest(pcapPacket); - return; - } - // Handle server connection initiation acknowledgement - if (tcpPacket.getHeader().getSyn() && tcpPacket.getHeader().getAck()) { - // A segment with both the SYN and ACK flags set indicates that the server has accepted the client's request - // to initiate a new connection. - processNewConnectionAck(pcapPacket); - return; - } - // Handle resets - if (tcpPacket.getHeader().getRst()) { - processRstPacket(pcapPacket); - return; - } - // Handle FINs - if (tcpPacket.getHeader().getFin()) { - // Handle FIN packet. - processFinPacket(pcapPacket); - } - // Handle ACKs (currently only ACKs of FINS) - if (tcpPacket.getHeader().getAck()) { - processAck(pcapPacket); - } - // Handle packets that carry payload (application data). - if (tcpPacket.getPayload() != null) { - processPayloadPacket(pcapPacket); - } - } - - private void processNewConnectionRequest(PcapPacket clientSynPacket) { - // A SYN w/o ACK always originates from the client. - Conversation conv = Conversation.fromPcapPacket(clientSynPacket, true); - conv.addSynPacket(clientSynPacket); - // Is there an ongoing conversation for the same four tuple (clientIp, clientPort, serverIp, serverPort) as - // found in the new SYN packet? - Conversation ongoingConv = mOpenConversations.get(conv); - if (ongoingConv != null) { - if (ongoingConv.isRetransmission(clientSynPacket)) { - // SYN retransmission detected, do nothing. - return; - // TODO: the way retransmission detection is implemented may cause a bug for connections where we have - // not recorded the initial SYN, but only the SYN ACK, as retransmission is determined by comparing the - // sequence numbers of initial SYNs -- and if no initial SYN is present for the Conversation, the new - // SYN will be interpreted as a retransmission. Possible fix: let isRentransmission ALWAYS return false - // when presented with a SYN packet when the Conversation already holds a SYN ACK packet? - } else { - // New SYN has different sequence number than SYN recorded for ongoingConv, so this must be an attempt - // to establish a new conversation with the same four tuple as ongoingConv. - // Mark existing connection as terminated. - // TODO: is this 100% theoretically correct, e.g., if many connection attempts are made back to back? And RST packets? - mTerminatedConversations.add(ongoingConv); - mOpenConversations.remove(ongoingConv); - } - } - // Finally, update the map of open connections with the new connection. - mOpenConversations.put(conv, conv); - } - - - /* - * TODO a problem across the board for all processXPacket methods below: - * if we start the capture in the middle of a TCP connection, we will not have an entry for the conversation in the - * map as we have not seen the initial SYN packet. - * Two ways we can address this: - * a) Perform null-checks and ignore packets for which we have not seen SYN - * + easy to get correct - * - we discard data (issue for long-lived connections!) - * b) Add a corresponding conversation entry whenever we encounter a packet that does not map to a conversation - * + we consider all data - * - not immediately clear if this will introduce bugs (incorrectly mapping packets to wrong conversations?) - * - * [[[ I went with option b) for now; see getOngoingConversationOrCreateNew(PcapPacket pcapPacket). ]]] - */ - - private void processNewConnectionAck(PcapPacket srvSynPacket) { - // Find the corresponding ongoing connection, if any (if we start the capture just *after* the initial SYN, no - // ongoing conversation entry will exist, so it must be created in that case). -// Conversation conv = mOpenConversations.get(Conversation.fromPcapPacket(srvSynPacket, false)); - Conversation conv = getOngoingConversationOrCreateNew(srvSynPacket); - // Note: exploits &&'s short-circuit operation: only attempts to add non-retransmissions. - if (!conv.isRetransmission(srvSynPacket) && !conv.addSynPacket(srvSynPacket)) { - // For safety/debugging: if NOT a retransmission and add fails, - // something has gone terribly wrong/invariant is broken. -// throw new AssertionError("Attempt to add SYN ACK packet that was NOT a retransmission failed." + -// Conversation.class.getSimpleName() + " invariant broken."); - } - } - - private void processRstPacket(PcapPacket rstPacket) { - Conversation conv = getOngoingConversationOrCreateNew(rstPacket); - // Add RST packet to conversation. - conv.addRstPacket(rstPacket); - // Move conversation to set of terminated conversations. - mTerminatedConversations.add(conv); - mOpenConversations.remove(conv, conv); - } - - private void processFinPacket(PcapPacket finPacket) { -// getOngoingConversationForPacket(finPacket).addFinPacket(finPacket); - getOngoingConversationOrCreateNew(finPacket).addFinPacket(finPacket); - } - - private void processAck(PcapPacket ackPacket) { -// getOngoingConversationForPacket(ackPacket).attemptAcknowledgementOfFin(ackPacket); - // Note that unlike the style for SYN, FIN, and payload packets, for "ACK only" packets, we want to avoid - // creating a new conversation. - Conversation conv = getOngoingConversationForPacket(ackPacket); - if (conv != null) { - // The ACK may be an ACK of a FIN, so attempt to mark the FIN as ack'ed. - conv.attemptAcknowledgementOfFin(ackPacket); - if (conv.isGracefullyShutdown()) { - // Move conversation to set of terminated conversations. - mTerminatedConversations.add(conv); - mOpenConversations.remove(conv); - } - } - // Note: add (additional) processing of ACKs (that are not ACKs of FINs) as necessary here... - } - - private void processPayloadPacket(PcapPacket pcapPacket) { -// getOngoingConversationForPacket(pcapPacket).addPacket(pcapPacket, true); - getOngoingConversationOrCreateNew(pcapPacket).addPacket(pcapPacket, true); - } - - /** - * Locates an ongoing conversation (if any) that {@code pcapPacket} pertains to. - * @param pcapPacket The packet that is to be mapped to an ongoing {@code Conversation}. - * @return The {@code Conversation} matching {@code pcapPacket} or {@code null} if there is no match. - */ - private Conversation getOngoingConversationForPacket(PcapPacket pcapPacket) { - // We cannot know if this is a client-to-server or server-to-client packet without trying both options... - Conversation conv = mOpenConversations.get(Conversation.fromPcapPacket(pcapPacket, true)); - if (conv == null) { - conv = mOpenConversations.get(Conversation.fromPcapPacket(pcapPacket, false)); - } - return conv; - } - - /** - * Like {@link #getOngoingConversationForPacket(PcapPacket)}, but creates and inserts a new {@code Conversation} - * into {@link #mOpenConversations} if no open conversation is found (i.e., in the case that - * {@link #getOngoingConversationForPacket(PcapPacket)} returns {@code null}). - * - * @param pcapPacket The packet that is to be mapped to an ongoing {@code Conversation}. - * @return The existing, ongoing {@code Conversation} matching {@code pcapPacket} or the newly created one in case - * no match was found. - */ - private Conversation getOngoingConversationOrCreateNew(PcapPacket pcapPacket) { - Conversation conv = getOngoingConversationForPacket(pcapPacket); - if (conv == null) { - TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class); - if (tcpPacket.getHeader().getSyn() && tcpPacket.getHeader().getAck()) { - // A SYN ACK packet always originates from the server (it is a reply to the initial SYN packet from the client) - conv = Conversation.fromPcapPacket(pcapPacket, false); - } else { - // TODO: can we do anything else but arbitrarily select who is designated as the server in this case? - // We can check if the IP prefix matches a local IP when handling traffic observed inside the local - // network, but that obviously won't be a useful strategy for an observer at the WAN port. - String srcIp = pcapPacket.get(IpV4Packet.class).getHeader().getSrcAddr().getHostAddress(); - // TODO: REPLACE THE ROUTER'S IP WITH A PARAMETER!!! - boolean clientIsSrc = srcIp.startsWith("10.") || srcIp.startsWith("192.168.") || srcIp.equals("128.195.205.105"); - conv = Conversation.fromPcapPacket(pcapPacket, clientIsSrc); - } - mOpenConversations.put(conv, conv); - } - return conv; - } -} diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TcpConversationUtils.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TcpConversationUtils.java index 7484d8a..f1c264b 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TcpConversationUtils.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TcpConversationUtils.java @@ -1,6 +1,6 @@ package edu.uci.iotproject.analysis; -import edu.uci.iotproject.Conversation; +import edu.uci.iotproject.trafficreassembly.layer3.Conversation; import edu.uci.iotproject.DnsMap; import edu.uci.iotproject.util.PcapPacketUtils; import org.pcap4j.core.PcapPacket; diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TrafficLabeler.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TrafficLabeler.java index ebc87b1..983de12 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TrafficLabeler.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TrafficLabeler.java @@ -1,8 +1,8 @@ package edu.uci.iotproject.analysis; -import edu.uci.iotproject.Conversation; +import edu.uci.iotproject.trafficreassembly.layer3.Conversation; import edu.uci.iotproject.DnsMap; -import edu.uci.iotproject.TcpReassembler; +import edu.uci.iotproject.trafficreassembly.layer3.TcpReassembler; import org.pcap4j.core.PacketListener; import org.pcap4j.core.PcapPacket; diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/AbstractPatternComparisonResult.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/AbstractPatternComparisonResult.java index f3bd585..cadadbe 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/AbstractPatternComparisonResult.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/AbstractPatternComparisonResult.java @@ -1,6 +1,6 @@ package edu.uci.iotproject.comparison; -import edu.uci.iotproject.Conversation; +import edu.uci.iotproject.trafficreassembly.layer3.Conversation; import edu.uci.iotproject.FlowPattern; /** diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/ComparisonFunctions.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/ComparisonFunctions.java index 15eda20..960d6a7 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/ComparisonFunctions.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/ComparisonFunctions.java @@ -1,6 +1,6 @@ package edu.uci.iotproject.comparison; -import edu.uci.iotproject.Conversation; +import edu.uci.iotproject.trafficreassembly.layer3.Conversation; import edu.uci.iotproject.FlowPattern; import org.pcap4j.core.PcapPacket; import org.pcap4j.packet.TcpPacket; diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/CompleteMatchPatternComparisonResult.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/CompleteMatchPatternComparisonResult.java index 3ddd884..960a0d0 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/CompleteMatchPatternComparisonResult.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/CompleteMatchPatternComparisonResult.java @@ -1,6 +1,6 @@ package edu.uci.iotproject.comparison; -import edu.uci.iotproject.Conversation; +import edu.uci.iotproject.trafficreassembly.layer3.Conversation; import edu.uci.iotproject.FlowPattern; /** diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/PatternComparisonTask.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/PatternComparisonTask.java index 808dee7..5a888b7 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/PatternComparisonTask.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/PatternComparisonTask.java @@ -1,6 +1,6 @@ package edu.uci.iotproject.comparison; -import edu.uci.iotproject.Conversation; +import edu.uci.iotproject.trafficreassembly.layer3.Conversation; import edu.uci.iotproject.FlowPattern; import java.util.concurrent.Callable; diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/seqalignment/ExtractedSequence.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/seqalignment/ExtractedSequence.java index 297107d..2d193a9 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/seqalignment/ExtractedSequence.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/seqalignment/ExtractedSequence.java @@ -1,6 +1,6 @@ package edu.uci.iotproject.comparison.seqalignment; -import edu.uci.iotproject.Conversation; +import edu.uci.iotproject.trafficreassembly.layer3.Conversation; import org.pcap4j.core.PcapPacket; import java.util.List; diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/seqalignment/SequenceExtraction.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/seqalignment/SequenceExtraction.java index c611968..6aaa318 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/seqalignment/SequenceExtraction.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/seqalignment/SequenceExtraction.java @@ -1,9 +1,8 @@ package edu.uci.iotproject.comparison.seqalignment; -import edu.uci.iotproject.Conversation; +import edu.uci.iotproject.trafficreassembly.layer3.Conversation; import edu.uci.iotproject.analysis.TcpConversationUtils; -import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.stream.Collectors; diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/AbstractClusterMatcher.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/AbstractClusterMatcher.java index 4c74eb8..3815acd 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/AbstractClusterMatcher.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/AbstractClusterMatcher.java @@ -1,6 +1,5 @@ package edu.uci.iotproject.detection; -import edu.uci.iotproject.Conversation; import org.pcap4j.core.PcapPacket; import java.util.List; diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/ClusterMatcher.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/ClusterMatcher.java index 25f3191..1ff2c02 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/ClusterMatcher.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/ClusterMatcher.java @@ -1,7 +1,7 @@ package edu.uci.iotproject.detection; -import edu.uci.iotproject.Conversation; -import edu.uci.iotproject.TcpReassembler; +import edu.uci.iotproject.trafficreassembly.layer3.Conversation; +import edu.uci.iotproject.trafficreassembly.layer3.TcpReassembler; import edu.uci.iotproject.analysis.TcpConversationUtils; import edu.uci.iotproject.io.PcapHandleReader; import edu.uci.iotproject.util.PrintUtils; diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/Layer2ClusterMatcher.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/Layer2ClusterMatcher.java deleted file mode 100644 index e0438d6..0000000 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/Layer2ClusterMatcher.java +++ /dev/null @@ -1,148 +0,0 @@ -package edu.uci.iotproject.detection; - -import edu.uci.iotproject.L2FlowReassembler; -import edu.uci.iotproject.Layer2Flow; -import edu.uci.iotproject.Layer2FlowReassemblerObserver; -import edu.uci.iotproject.io.PcapHandleReader; -import edu.uci.iotproject.util.PrintUtils; -import org.pcap4j.core.*; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * TODO add class documentation. - * - * @author Janus Varmarken - */ -public class Layer2ClusterMatcher extends AbstractClusterMatcher implements Layer2FlowReassemblerObserver, Layer2FlowObserver { - - public static void main(String[] args) throws PcapNativeException, NotOpenException { - final String onSignatureFile = "/Users/varmarken/temp/UCI IoT Project/experiments/training/signatures/tplink-plug/tplink-plug-onSignature-device-side.sig"; - List>> onSignature = PrintUtils.deserializeSignatureFromFile(onSignatureFile); - - - L2FlowReassembler flowReassembler = new L2FlowReassembler(); - - Layer2ClusterMatcher l2ClusterMatcher = new Layer2ClusterMatcher(onSignature.get(0)); - flowReassembler.addObserver(l2ClusterMatcher); - - final String inputPcapFile = "/Users/varmarken/temp/UCI IoT Project/experiments/2018-07/tplink/tplink.wlan1.local.pcap"; - - PcapHandle handle; - try { - handle = Pcaps.openOffline(inputPcapFile, PcapHandle.TimestampPrecision.NANO); - } catch (PcapNativeException pne) { - handle = Pcaps.openOffline(inputPcapFile); - } - PcapHandleReader reader = new PcapHandleReader(handle, p -> true, flowReassembler); - reader.readFromHandle(); - - - } - - - private final List mSeqMatchers; - - public Layer2ClusterMatcher(List> cluster) { - super(cluster); - // Setup a sequence matcher for each sequence of the pruned cluster - mSeqMatchers = new ArrayList<>(); - mCluster.forEach(seq -> mSeqMatchers.add(new Layer2SequenceMatcher(seq))); - -// for (int i = 0; i < mCluster.size(); i++) { -// -// -// mSeqMatchers[i] = new Layer2SequenceMatcher(mCluster.get(i)); -// -// -// } - } - -// @Override -// public void gotPacket(PcapPacket packet) { -// // Forward the packet to all sequence matchers. -// for (Layer2SequenceMatcher matcher : mSeqMatchers) { -// matcher.gotPacket(packet); -// } -// -// -// } - - - private final Map> mPerFlowSeqMatchers = new HashMap<>(); - - @Override - public void onNewPacket(Layer2Flow flow, PcapPacket newPacket) { - if (mPerFlowSeqMatchers.get(flow) == null) { - // If this is the first time we encounter this flow, we need to set up sequence matchers for it. - List matchers = new ArrayList<>(); - mCluster.forEach(seq -> matchers.add(new Layer2SequenceMatcher(seq))); - mPerFlowSeqMatchers.put(flow, matchers); - } - // Buffer for new sequence matchers that will take over the job of observing for the first packet when a - // sequence matcher advances beyond the first packet. - List newSeqMatchers = new ArrayList<>(); - // Buffer for sequence matchers that have terminated and are to be removed from mPerFlowSeqMatchers. - List terminatedSeqMatchers = new ArrayList<>(); - // Present the new packet to all sequence matchers - for (Layer2SequenceMatcher sm : mPerFlowSeqMatchers.get(flow)) { - boolean matched = sm.matchPacket(newPacket); - if (matched && sm.getMatchedPacketsCount() == 1) { - // Setup a new sequence matcher that matches from the beginning of the sequence so as to keep - // progressing in the sequence matcher that just matched the current packet, while still allowing - // for matches of the full sequence in later traffic. This is to accommodate the case where the - // first packet of a sequence is detected in an early packet, but where the remaining packets of - // that sequence do not appear until way later in time (e.g., if the first packet of the sequence - // by chance is generated from traffic unrelated to the trigger traffic). - // Note that we must store the new sequence matcher in a buffer and add it outside the loop in order to - // prevent concurrent modification exceptions. - newSeqMatchers.add(new Layer2SequenceMatcher(sm.getTargetSequence())); - } - if (matched && sm.getMatchedPacketsCount() == sm.getTargetSequencePacketCount()) { - // This sequence matcher has a match of the sequence it was searching for - // TODO report it.... for now just do a dummy printout. - System.out.println("SEQUENCE MATCHER HAS A MATCH AT " + sm.getMatchedPackets().get(0).getTimestamp()); - // Mark the sequence matcher for removal. No need to create a replacement one as we do that whenever the - // first packet of the sequence is matched (see above). - terminatedSeqMatchers.add(sm); - } - } - // Add the new sequence matchers, if any. - mPerFlowSeqMatchers.get(flow).addAll(newSeqMatchers); - // Remove the terminated sequence matchers, if any. - mPerFlowSeqMatchers.get(flow).removeAll(terminatedSeqMatchers); - } - - - @Override - protected List> pruneCluster(List> cluster) { - // Note: we assume that all sequences in the input cluster are of the same length and that their packet - // directions are identical. - List> prunedCluster = new ArrayList<>(); - for (List originalClusterSeq : cluster) { - boolean alreadyPresent = prunedCluster.stream().anyMatch(pcPkts -> { - for (int i = 0; i < pcPkts.size(); i++) { - if (pcPkts.get(i).getOriginalLength() != originalClusterSeq.get(i).getOriginalLength()) { - return false; - } - } - return true; - }); - if (!alreadyPresent) { - // Add the sequence if not already present in the pruned cluster. - prunedCluster.add(originalClusterSeq); - } - } - return prunedCluster; - } - - - @Override - public void onNewFlow(L2FlowReassembler reassembler, Layer2Flow newFlow) { - // Subscribe to the new flow to get updates whenever a new packet pertaining to the flow is processed. - newFlow.addFlowObserver(this); - } -} diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/Layer2FlowObserver.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/Layer2FlowObserver.java index 05cf9a1..26823de 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/Layer2FlowObserver.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/Layer2FlowObserver.java @@ -1,6 +1,6 @@ package edu.uci.iotproject.detection; -import edu.uci.iotproject.Layer2Flow; +import edu.uci.iotproject.trafficreassembly.layer2.Layer2Flow; import org.pcap4j.core.PcapPacket; /** diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/Layer2SequenceMatcher.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/Layer2SequenceMatcher.java deleted file mode 100644 index 35d628d..0000000 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/Layer2SequenceMatcher.java +++ /dev/null @@ -1,76 +0,0 @@ -package edu.uci.iotproject.detection; - -import edu.uci.iotproject.analysis.TriggerTrafficExtractor; -import org.pcap4j.core.PcapPacket; - -import java.util.ArrayList; -import java.util.List; - -/** - * TODO add class documentation. - * - * @author Janus Varmarken - */ -public class Layer2SequenceMatcher { - - /** - * The sequence this {@link Layer2SequenceMatcher} is searching for. - */ - private final List mSequence; - - /** - * Buffer of actual packets seen so far that match the searched sequence (i.e., constitutes a subsequence of the - * searched sequence). - */ - private final List mMatchedPackets = new ArrayList<>(); - - public Layer2SequenceMatcher(List sequence) { - mSequence = sequence; - } - - public boolean matchPacket(PcapPacket packet) { - // The packet we want to match next. - PcapPacket expected = mSequence.get(mMatchedPackets.size()); - // First verify if the received packet has the length we're looking for. - if (packet.getOriginalLength() == expected.getOriginalLength()) { - // Next apply timing constraints: - // - to be a match, the packet must have a later timestamp than any other packet currently matched - // - does adding the packet cause the max allowed time between first packet and last packet to be exceeded? - if (mMatchedPackets.size() > 0 && - !packet.getTimestamp().isAfter(mMatchedPackets.get(mMatchedPackets.size()-1).getTimestamp())) { - return false; - } - if (mMatchedPackets.size() > 0 && - packet.getTimestamp(). - isAfter(mMatchedPackets.get(0).getTimestamp(). - plusMillis(TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS))) { - // Packet too - return false; - } - // TODO (how to) check directions? - // This packet has a length matching next packet of searched sequence, so we store it and advance. - mMatchedPackets.add(packet); - if (mMatchedPackets.size() == mSequence.size()) { - // TODO report (to observers?) that we are done. - } - return true; - } - return false; - } - - public int getMatchedPacketsCount() { - return mMatchedPackets.size(); - } - - public int getTargetSequencePacketCount() { - return mSequence.size(); - } - - public List getTargetSequence() { - return mSequence; - } - - public List getMatchedPackets() { - return mMatchedPackets; - } -} diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer2/Layer2ClusterMatcher.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer2/Layer2ClusterMatcher.java new file mode 100644 index 0000000..d50a517 --- /dev/null +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer2/Layer2ClusterMatcher.java @@ -0,0 +1,150 @@ +package edu.uci.iotproject.detection.layer2; + +import edu.uci.iotproject.trafficreassembly.layer2.Layer2FlowReassembler; +import edu.uci.iotproject.trafficreassembly.layer2.Layer2Flow; +import edu.uci.iotproject.trafficreassembly.layer2.Layer2FlowReassemblerObserver; +import edu.uci.iotproject.detection.AbstractClusterMatcher; +import edu.uci.iotproject.detection.Layer2FlowObserver; +import edu.uci.iotproject.io.PcapHandleReader; +import edu.uci.iotproject.util.PrintUtils; +import org.pcap4j.core.*; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * TODO add class documentation. + * + * @author Janus Varmarken + */ +public class Layer2ClusterMatcher extends AbstractClusterMatcher implements Layer2FlowReassemblerObserver, Layer2FlowObserver { + + public static void main(String[] args) throws PcapNativeException, NotOpenException { + final String onSignatureFile = "/Users/varmarken/temp/UCI IoT Project/experiments/training/signatures/tplink-plug/tplink-plug-onSignature-device-side.sig"; + List>> onSignature = PrintUtils.deserializeSignatureFromFile(onSignatureFile); + + + Layer2FlowReassembler flowReassembler = new Layer2FlowReassembler(); + + Layer2ClusterMatcher l2ClusterMatcher = new Layer2ClusterMatcher(onSignature.get(0)); + flowReassembler.addObserver(l2ClusterMatcher); + + final String inputPcapFile = "/Users/varmarken/temp/UCI IoT Project/experiments/2018-07/tplink/tplink.wlan1.local.pcap"; + + PcapHandle handle; + try { + handle = Pcaps.openOffline(inputPcapFile, PcapHandle.TimestampPrecision.NANO); + } catch (PcapNativeException pne) { + handle = Pcaps.openOffline(inputPcapFile); + } + PcapHandleReader reader = new PcapHandleReader(handle, p -> true, flowReassembler); + reader.readFromHandle(); + + + } + + + private final List mSeqMatchers; + + public Layer2ClusterMatcher(List> cluster) { + super(cluster); + // Setup a sequence matcher for each sequence of the pruned cluster + mSeqMatchers = new ArrayList<>(); + mCluster.forEach(seq -> mSeqMatchers.add(new Layer2SequenceMatcher(seq))); + +// for (int i = 0; i < mCluster.size(); i++) { +// +// +// mSeqMatchers[i] = new Layer2SequenceMatcher(mCluster.get(i)); +// +// +// } + } + +// @Override +// public void gotPacket(PcapPacket packet) { +// // Forward the packet to all sequence matchers. +// for (Layer2SequenceMatcher matcher : mSeqMatchers) { +// matcher.gotPacket(packet); +// } +// +// +// } + + + private final Map> mPerFlowSeqMatchers = new HashMap<>(); + + @Override + public void onNewPacket(Layer2Flow flow, PcapPacket newPacket) { + if (mPerFlowSeqMatchers.get(flow) == null) { + // If this is the first time we encounter this flow, we need to set up sequence matchers for it. + List matchers = new ArrayList<>(); + mCluster.forEach(seq -> matchers.add(new Layer2SequenceMatcher(seq))); + mPerFlowSeqMatchers.put(flow, matchers); + } + // Buffer for new sequence matchers that will take over the job of observing for the first packet when a + // sequence matcher advances beyond the first packet. + List newSeqMatchers = new ArrayList<>(); + // Buffer for sequence matchers that have terminated and are to be removed from mPerFlowSeqMatchers. + List terminatedSeqMatchers = new ArrayList<>(); + // Present the new packet to all sequence matchers + for (Layer2SequenceMatcher sm : mPerFlowSeqMatchers.get(flow)) { + boolean matched = sm.matchPacket(newPacket); + if (matched && sm.getMatchedPacketsCount() == 1) { + // Setup a new sequence matcher that matches from the beginning of the sequence so as to keep + // progressing in the sequence matcher that just matched the current packet, while still allowing + // for matches of the full sequence in later traffic. This is to accommodate the case where the + // first packet of a sequence is detected in an early packet, but where the remaining packets of + // that sequence do not appear until way later in time (e.g., if the first packet of the sequence + // by chance is generated from traffic unrelated to the trigger traffic). + // Note that we must store the new sequence matcher in a buffer and add it outside the loop in order to + // prevent concurrent modification exceptions. + newSeqMatchers.add(new Layer2SequenceMatcher(sm.getTargetSequence())); + } + if (matched && sm.getMatchedPacketsCount() == sm.getTargetSequencePacketCount()) { + // This sequence matcher has a match of the sequence it was searching for + // TODO report it.... for now just do a dummy printout. + System.out.println("SEQUENCE MATCHER HAS A MATCH AT " + sm.getMatchedPackets().get(0).getTimestamp()); + // Mark the sequence matcher for removal. No need to create a replacement one as we do that whenever the + // first packet of the sequence is matched (see above). + terminatedSeqMatchers.add(sm); + } + } + // Add the new sequence matchers, if any. + mPerFlowSeqMatchers.get(flow).addAll(newSeqMatchers); + // Remove the terminated sequence matchers, if any. + mPerFlowSeqMatchers.get(flow).removeAll(terminatedSeqMatchers); + } + + + @Override + protected List> pruneCluster(List> cluster) { + // Note: we assume that all sequences in the input cluster are of the same length and that their packet + // directions are identical. + List> prunedCluster = new ArrayList<>(); + for (List originalClusterSeq : cluster) { + boolean alreadyPresent = prunedCluster.stream().anyMatch(pcPkts -> { + for (int i = 0; i < pcPkts.size(); i++) { + if (pcPkts.get(i).getOriginalLength() != originalClusterSeq.get(i).getOriginalLength()) { + return false; + } + } + return true; + }); + if (!alreadyPresent) { + // Add the sequence if not already present in the pruned cluster. + prunedCluster.add(originalClusterSeq); + } + } + return prunedCluster; + } + + + @Override + public void onNewFlow(Layer2FlowReassembler reassembler, Layer2Flow newFlow) { + // Subscribe to the new flow to get updates whenever a new packet pertaining to the flow is processed. + newFlow.addFlowObserver(this); + } +} diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SequenceMatcher.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SequenceMatcher.java new file mode 100644 index 0000000..de5c14a --- /dev/null +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SequenceMatcher.java @@ -0,0 +1,76 @@ +package edu.uci.iotproject.detection.layer2; + +import edu.uci.iotproject.analysis.TriggerTrafficExtractor; +import org.pcap4j.core.PcapPacket; + +import java.util.ArrayList; +import java.util.List; + +/** + * TODO add class documentation. + * + * @author Janus Varmarken + */ +public class Layer2SequenceMatcher { + + /** + * The sequence this {@link Layer2SequenceMatcher} is searching for. + */ + private final List mSequence; + + /** + * Buffer of actual packets seen so far that match the searched sequence (i.e., constitutes a subsequence of the + * searched sequence). + */ + private final List mMatchedPackets = new ArrayList<>(); + + public Layer2SequenceMatcher(List sequence) { + mSequence = sequence; + } + + public boolean matchPacket(PcapPacket packet) { + // The packet we want to match next. + PcapPacket expected = mSequence.get(mMatchedPackets.size()); + // First verify if the received packet has the length we're looking for. + if (packet.getOriginalLength() == expected.getOriginalLength()) { + // Next apply timing constraints: + // - to be a match, the packet must have a later timestamp than any other packet currently matched + // - does adding the packet cause the max allowed time between first packet and last packet to be exceeded? + if (mMatchedPackets.size() > 0 && + !packet.getTimestamp().isAfter(mMatchedPackets.get(mMatchedPackets.size()-1).getTimestamp())) { + return false; + } + if (mMatchedPackets.size() > 0 && + packet.getTimestamp(). + isAfter(mMatchedPackets.get(0).getTimestamp(). + plusMillis(TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS))) { + // Packet too + return false; + } + // TODO (how to) check directions? + // This packet has a length matching next packet of searched sequence, so we store it and advance. + mMatchedPackets.add(packet); + if (mMatchedPackets.size() == mSequence.size()) { + // TODO report (to observers?) that we are done. + } + return true; + } + return false; + } + + public int getMatchedPacketsCount() { + return mMatchedPackets.size(); + } + + public int getTargetSequencePacketCount() { + return mSequence.size(); + } + + public List getTargetSequence() { + return mSequence; + } + + public List getMatchedPackets() { + return mMatchedPackets; + } +} diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/evaluation/SanitySignatureGenerator.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/evaluation/SanitySignatureGenerator.java index 9cecf56..060387a 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/evaluation/SanitySignatureGenerator.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/evaluation/SanitySignatureGenerator.java @@ -1,7 +1,7 @@ package edu.uci.iotproject.evaluation; -import edu.uci.iotproject.Conversation; -import edu.uci.iotproject.TcpReassembler; +import edu.uci.iotproject.trafficreassembly.layer3.Conversation; +import edu.uci.iotproject.trafficreassembly.layer3.TcpReassembler; import edu.uci.iotproject.io.PcapHandleReader; import edu.uci.iotproject.util.PrintUtils; import org.pcap4j.core.*; diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2Flow.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2Flow.java new file mode 100644 index 0000000..f1b7190 --- /dev/null +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2Flow.java @@ -0,0 +1,79 @@ +package edu.uci.iotproject.trafficreassembly.layer2; + +import edu.uci.iotproject.detection.Layer2FlowObserver; +import org.pcap4j.core.PcapPacket; +import org.pcap4j.packet.EthernetPacket; +import org.pcap4j.util.MacAddress; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * The packets exchanged between two endpoints (MAC addresses). + * + * @author Janus Varmarken + */ +public class Layer2Flow { + + private final MacAddress mEndpoint1; + private final MacAddress mEndpoint2; + + private final List mFlowObservers = new ArrayList<>(); + + public Layer2Flow(MacAddress endpoint1, MacAddress endpoint2) { + mEndpoint1 = endpoint1; + mEndpoint2 = endpoint2; + } + + public void addFlowObserver(Layer2FlowObserver observer) { + mFlowObservers.add(observer); + } + + public void removeFlowObserver(Layer2FlowObserver observer) { + mFlowObservers.remove(observer); + } + + /** + * The packets in the flow. + */ + private final List mPackets = new ArrayList<>(); + + /** + * Add a packet to this flow. + * @param packet The packet that is to be added to the flow. + */ + public void addPacket(PcapPacket packet) { + verifyAddresses(packet); + mPackets.add(packet); + // Notify flow observers of the new packet + mFlowObservers.forEach(o -> o.onNewPacket(this, packet)); + } + + public List getPackets() { + return Collections.unmodifiableList(mPackets); + } + + private void verifyAddresses(PcapPacket packet) { + EthernetPacket ethPkt = packet.get(EthernetPacket.class); + MacAddress srcAddr = ethPkt.getHeader().getSrcAddr(); + MacAddress dstAddr = ethPkt.getHeader().getDstAddr(); + if ((mEndpoint1.equals(srcAddr) && mEndpoint2.equals(dstAddr)) || + (mEndpoint1.equals(dstAddr) && mEndpoint2.equals(srcAddr))) { + // All is good. + return; + } + throw new IllegalArgumentException("Mismatch in MACs: packet does not pertain to this flow"); + } + +} + + + +/* + + + Packet stream -> flow reassembler -> flow1, flow2, flow3... -> for each flow, keep a sequence matcher for each sequence of cluster + + + */ \ No newline at end of file diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowReassembler.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowReassembler.java new file mode 100644 index 0000000..e7b7304 --- /dev/null +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowReassembler.java @@ -0,0 +1,86 @@ +package edu.uci.iotproject.trafficreassembly.layer2; + +import edu.uci.iotproject.trafficreassembly.layer2.Layer2Flow; +import edu.uci.iotproject.trafficreassembly.layer2.Layer2FlowReassemblerObserver; +import org.pcap4j.core.PacketListener; +import org.pcap4j.core.PcapPacket; +import org.pcap4j.packet.EthernetPacket; +import org.pcap4j.util.MacAddress; + +import java.util.*; + +/** + * Reassembles traffic flows at layer 2, i.e., for each combination of hosts, creates a list of packets exchanged + * between said hosts. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class Layer2FlowReassembler implements PacketListener { + + /** + * Maps a pair of MAC addresses to the packets exchanged between the two hosts. + * The key is the concatenation of the two MAC addresses in hex string format, where the lexicographically smaller + * MAC is at the front of the string. + */ + private final Map mFlows = new HashMap<>(); + + private final List mObservers = new ArrayList<>(); + + @Override + public void gotPacket(PcapPacket packet) { + // TODO: update to 802.11 packet...? + EthernetPacket ethPkt = packet.get(EthernetPacket.class); + + MacAddress srcAddr = ethPkt.getHeader().getSrcAddr(); + MacAddress dstAddr = ethPkt.getHeader().getDstAddr(); + + String key = keyFromAddresses(srcAddr, dstAddr); + // Create a new list if this pair of MAC addresses where not previously encountered and add packet to that list, + // or simply add to an existing list if one is present. + mFlows.computeIfAbsent(key, k -> { + Layer2Flow newFlow = new Layer2Flow(srcAddr, dstAddr); + // Inform observers of the new flow + mObservers.forEach(o -> o.onNewFlow(this, newFlow)); + return newFlow; + }).addPacket(packet); + } + + public void addObserver(Layer2FlowReassemblerObserver observer) { + mObservers.add(observer); + } + + public void removeObserver(Layer2FlowReassemblerObserver observer) { + mObservers.remove(observer); + } + + /** + * Get the traffic flow between two local endpoints ({@link MacAddress}es). + * @param addr1 The first endpoint. + * @param addr2 The second endpoint + * @return The traffic exchanged between the two endpoints. + */ + public Layer2Flow getFlowForAddresses(MacAddress addr1, MacAddress addr2) { + return mFlows.get(keyFromAddresses(addr1, addr2)); + } + + /** + * Get all traffic flows, i.e., a traffic flow for each unique pair of endpoints (MAC addresses). + * @return All traffic flows. + */ + public Collection getFlows() { + return mFlows.values(); + } + + /** + * Given two {@link MacAddress}es, generates the corresponding key string used in {@link #mFlows}. + * @param addr1 The first address. + * @param addr2 The second address. + * @return the key string used in {@link #mFlows} corresponding to the two addresses. + */ + private String keyFromAddresses(MacAddress addr1, MacAddress addr2) { + String addr1Str = addr1.toString(); + String addr2Str = addr2.toString(); + return addr1Str.compareTo(addr2Str) < 0 ? addr1Str + addr2Str : addr2Str + addr1Str; + } +} diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowReassemblerObserver.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowReassemblerObserver.java new file mode 100644 index 0000000..a71cd19 --- /dev/null +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowReassemblerObserver.java @@ -0,0 +1,19 @@ +package edu.uci.iotproject.trafficreassembly.layer2; + +/** + * For observing a {@link Layer2FlowReassembler}. + * + * @author Janus Varmarken + */ +public interface Layer2FlowReassemblerObserver { + + /** + * Invoked when when a {@link Layer2FlowReassembler} detects a new flow (i.e., when it encounters traffic between two + * MAC addresses that has not previously communicated in the traffic trace). + * + * @param reassembler The {@link Layer2FlowReassembler} that detected the new flow. + * @param newFlow The new flow. + */ + void onNewFlow(Layer2FlowReassembler reassembler, Layer2Flow newFlow); + +} diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/Conversation.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/Conversation.java new file mode 100644 index 0000000..e89e81b --- /dev/null +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/Conversation.java @@ -0,0 +1,585 @@ +package edu.uci.iotproject.trafficreassembly.layer3; + +import edu.uci.iotproject.analysis.TcpConversationUtils; +import edu.uci.iotproject.util.PcapPacketUtils; +import org.pcap4j.core.PcapPacket; +import org.pcap4j.packet.IpV4Packet; +import org.pcap4j.packet.Packet; +import org.pcap4j.packet.TcpPacket; + +import java.util.*; + +/** + * Models a (TCP) conversation/connection/session/flow (packet's belonging to the same session between a client and a + * server). + * Holds a list of {@link PcapPacket}s identified as pertaining to the flow. Note that this list is not + * considered when determining equality of two {@code Conversation} instances in order to allow for a + * {@code Conversation} to function as a key in data structures such as {@link java.util.Map} and {@link java.util.Set}. + * See {@link #equals(Object)} for the definition of equality. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class Conversation { + + /* Begin instance properties */ + /** + * The IP of the host that is considered the client (i.e. the host that initiates the conversation) + * in this conversation. + */ + private final String mClientIp; + + /** + * The port number used by the host that is considered the client in this conversation. + */ + private final int mClientPort; + + /** + * The IP of the host that is considered the server (i.e. is the responder) in this conversation. + */ + private final String mServerIp; + + /** + * The port number used by the server in this conversation. + */ + private final int mServerPort; + + /** + * The list of packets (with payload) pertaining to this conversation. + */ + private final List mPackets; + + /** + * If {@link #isTls()} is {@code true}, this list contains the subset of {@link #mPackets} which are TLS Application + * Data packets. + */ + private final List mTlsApplicationDataPackets; + + /** + * Contains the sequence numbers used thus far by the host that is considered the client in this + * {@code Conversation}. + * Used for filtering out retransmissions. + */ + private final Set mSeqNumbersClient; + + /** + * Contains the sequence numbers used thus far by the host that is considered the server in this + * {@code Conversation}. + * Used for filtering out retransmissions. + */ + private final Set mSeqNumbersSrv; + + /** + * List of SYN packets pertaining to this conversation. + */ + private final List mSynPackets; + + /** + * List of pairs FINs and their corresponding ACKs associated with this conversation. + */ + private final List mFinPackets; + + /** + * List of RST packets associated with this conversation. + */ + private final List mRstPackets; + + /** + * Boolean to mark the packet as Application Data based on the previous packet that reaches MTU + */ + private boolean mApplicationData; + /* End instance properties */ + + /** + * Factory method for creating a {@code Conversation} from a {@link PcapPacket}. + * @param pcapPacket The {@code PcapPacket} that wraps a TCP segment for which a {@code Conversation} is to be initiated. + * @param clientIsSrc If {@code true}, the source address and source port found in the IP datagram and TCP segment + * wrapped in the {@code PcapPacket} are regarded as pertaining to the client, and the destination + * address and destination port are regarded as pertaining to the server---and vice versa if set + * to {@code false}. + * @return A {@code Conversation} initiated with ip:port for client and server according to the direction of the packet. + */ + public static Conversation fromPcapPacket(PcapPacket pcapPacket, boolean clientIsSrc) { + IpV4Packet ipPacket = pcapPacket.get(IpV4Packet.class); + TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class); + String clientIp = clientIsSrc ? ipPacket.getHeader().getSrcAddr().getHostAddress() : + ipPacket.getHeader().getDstAddr().getHostAddress(); + String srvIp = clientIsSrc ? ipPacket.getHeader().getDstAddr().getHostAddress() : + ipPacket.getHeader().getSrcAddr().getHostAddress(); + int clientPort = clientIsSrc ? tcpPacket.getHeader().getSrcPort().valueAsInt() : + tcpPacket.getHeader().getDstPort().valueAsInt(); + int srvPort = clientIsSrc ? tcpPacket.getHeader().getDstPort().valueAsInt() : + tcpPacket.getHeader().getSrcPort().valueAsInt(); + return new Conversation(clientIp, clientPort, srvIp, srvPort); + } + + /** + * Constructs a new {@code Conversation}. + * @param clientIp The IP of the host that is considered the client (i.e. the host that initiates the conversation) + * in the conversation. + * @param clientPort The port number used by the client for the conversation. + * @param serverIp The IP of the host that is considered the server (i.e. is the responder) in the conversation. + * @param serverPort The port number used by the server for the conversation. + */ + public Conversation(String clientIp, int clientPort, String serverIp, int serverPort) { + this.mClientIp = clientIp; + this.mClientPort = clientPort; + this.mServerIp = serverIp; + this.mServerPort = serverPort; + this.mPackets = new ArrayList<>(); + this.mTlsApplicationDataPackets = new ArrayList<>(); + this.mSeqNumbersClient = new HashSet<>(); + this.mSeqNumbersSrv = new HashSet<>(); + this.mSynPackets = new ArrayList<>(); + this.mFinPackets = new ArrayList<>(); + this.mRstPackets = new ArrayList<>(); + this.mApplicationData = false; + } + + /** + * Add a packet to the list of packets associated with this conversation. + * @param packet The packet that is to be added to (associated with) this conversation. + * @param ignoreRetransmissions Boolean value indicating if retransmissions should be ignored. + * If set to {@code true}, {@code packet} will not be added to the + * internal list of packets pertaining to this {@code Conversation} + * iff the sequence number of {@code packet} was already + * seen in a previous packet. + */ + public void addPacket(PcapPacket packet, boolean ignoreRetransmissions) { + // Precondition: verify that packet does indeed pertain to conversation. + onAddPrecondition(packet); + if (ignoreRetransmissions && isRetransmission(packet)) { + // Packet is a retransmission. Ignore it. + return; + } + // Select direction-dependent set of sequence numbers seen so far and update it with sequence number of new packet. + addSeqNumber(packet); + // Finally add packet to list of packets pertaining to this conversation. + mPackets.add(packet); + // Preserve order of packets in list: sort according to timestamp. + if (mPackets.size() > 1 && + mPackets.get(mPackets.size()-1).getTimestamp().isBefore(mPackets.get(mPackets.size()-2).getTimestamp())) { + Collections.sort(mPackets, (o1, o2) -> { + if (o1.getTimestamp().isBefore(o2.getTimestamp())) { return -1; } + else if (o2.getTimestamp().isBefore(o1.getTimestamp())) { return 1; } + else { return 0; } + }); + } + // If TLS, inspect packet to see if it's a TLS Application Data packet, and if so add it to the list of TLS + // Application Data packets. + if (isTls()) { + TcpPacket tcpPacket = packet.get(TcpPacket.class); + Packet tcpPayload = tcpPacket.getPayload(); + if (tcpPayload == null) { + return; + } + byte[] rawPayload = tcpPayload.getRawData(); + // The SSL record header is at the front of the payload and is 5 bytes long. + // The SSL record header type field (the first byte) is set to 23 if it is an Application Data packet. + if (rawPayload != null && rawPayload.length >= 5) { + if (rawPayload[0] == 23) { + mTlsApplicationDataPackets.add(packet); + // Consider the following packet a data packet if this packet's size == MTU size 1448 + if (rawPayload.length >= 1448) + mApplicationData = true; + } else if (rawPayload[0] == 20) { + // Do nothing for now - CHANGE_CIPHER_SPEC + } else if (rawPayload[0] == 21) { + // Do nothing for now - ALERT + } else if (rawPayload[0] == 22) { + // Do nothing for now - HANDSHAKE + } else { + // If it is TLS with payload, but rawPayload[0] != 23 + if (mApplicationData == true) { + // It is a continuation of the previous packet if the previous packet reaches MTU size 1448 and + // it is not either type 20, 21, or 22 + mTlsApplicationDataPackets.add(packet); + if (rawPayload.length < 1448) + mApplicationData = false; + } + } + } + } + } + + /** + * Get a list of packets pertaining to this {@code Conversation}. + * The returned list is a read-only list. + * @return the list of packets pertaining to this {@code Conversation}. + */ + public List getPackets() { + // Return read-only view to prevent external code from manipulating internal state (preserve invariant). + return Collections.unmodifiableList(mPackets); + } + + /** + * Records a TCP SYN packet as pertaining to this conversation (adds it to the the internal list). + * Attempts to add duplicate SYN packets will be ignored, and the caller is made aware of the attempt to add a + * duplicate by the return value being {@code false}. + * + * @param synPacket A {@link PcapPacket} wrapping a TCP SYN packet. + * @return {@code true} if the packet was successfully added to this {@code Conversation}, {@code false} otherwise. + */ + public boolean addSynPacket(PcapPacket synPacket) { + onAddPrecondition(synPacket); + final IpV4Packet synPacketIpSection = synPacket.get(IpV4Packet.class); + final TcpPacket synPacketTcpSection = synPacket.get(TcpPacket.class); + if (synPacketTcpSection == null || !synPacketTcpSection.getHeader().getSyn()) { + throw new IllegalArgumentException("Not a SYN packet."); + } + // We are only interested in recording one copy of the two SYN packets (one SYN packet in each direction), i.e., + // we want to discard retransmitted SYN packets. + if (mSynPackets.size() >= 2) { + return false; + } + // Check the set of recorded SYN packets to see if we have already recorded a SYN packet going in the same + // direction as the packet given in the argument. + boolean matchingPrevSyn = mSynPackets.stream().anyMatch(p -> { + IpV4Packet pIp = p.get(IpV4Packet.class); + TcpPacket pTcp = p.get(TcpPacket.class); + boolean srcAddrMatch = synPacketIpSection.getHeader().getSrcAddr().getHostAddress(). + equals(pIp.getHeader().getSrcAddr().getHostAddress()); + boolean dstAddrMatch = synPacketIpSection.getHeader().getDstAddr().getHostAddress(). + equals(pIp.getHeader().getDstAddr().getHostAddress()); + boolean srcPortMatch = synPacketTcpSection.getHeader().getSrcPort().valueAsInt() == + pTcp.getHeader().getSrcPort().valueAsInt(); + boolean dstPortMatch = synPacketTcpSection.getHeader().getDstPort().valueAsInt() == + pTcp.getHeader().getDstPort().valueAsInt(); + return srcAddrMatch && dstAddrMatch && srcPortMatch && dstPortMatch; + }); + if (matchingPrevSyn) { + return false; + } + // Update direction-dependent set of sequence numbers and record/log packet. + addSeqNumber(synPacket); + return mSynPackets.add(synPacket); + + /* + mSynPackets.stream().anyMatch(p -> { + IpV4Packet pIp = p.get(IpV4Packet.class); + TcpPacket pTcp = p.get(TcpPacket.class); + boolean srcAddrMatch = synPacketIpSection.getHeader().getSrcAddr().getHostAddress(). + equals(pIp.getHeader().getSrcAddr().getHostAddress()); + boolean dstAddrMatch = synPacketIpSection.getHeader().getDstAddr().getHostAddress(). + equals(pIp.getHeader().getDstAddr().getHostAddress()); + boolean srcPortMatch = synPacketTcpSection.getHeader().getSrcPort().valueAsInt() == + pTcp.getHeader().getSrcPort().valueAsInt(); + boolean dstPortMatch = synPacketTcpSection.getHeader().getDstPort().value() == + pTcp.getHeader().getDstPort().value(); + + boolean fourTupleMatch = srcAddrMatch && dstAddrMatch && srcPortMatch && dstPortMatch; + + boolean seqNoMatch = synPacketTcpSection.getHeader().getSequenceNumber() == + pTcp.getHeader().getSequenceNumber(); + + if (fourTupleMatch && !seqNoMatch) { + // If the four tuple that identifies the conversation matches, but the sequence number is different, + // it means that this SYN packet is, in fact, an attempt to establish a **new** connection, and hence + // the given packet is NOT part of this conversation, even though the ip:port combinations are (by + // chance) selected such that they match this conversation. + throw new IllegalArgumentException("Attempt to add SYN packet that belongs to a different conversation " + + "(which is identified by the same four tuple as this conversation)"); + } + return fourTupleMatch && seqNoMatch; + }); + */ + } + + /** + * Get a list of SYN packets pertaining to this {@code Conversation}. + * The returned list is a read-only list. + * @return the list of SYN packets pertaining to this {@code Conversation}. + */ + public List getSynPackets() { + return Collections.unmodifiableList(mSynPackets); + } + + /** + * Adds a TCP FIN packet to the list of TCP FIN packets associated with this conversation. + * @param finPacket The TCP FIN packet that is to be added to (associated with) this conversation. + */ + public void addFinPacket(PcapPacket finPacket) { + // Precondition: verify that packet does indeed pertain to conversation. + onAddPrecondition(finPacket); + // TODO: should call addSeqNumber here? + addSeqNumber(finPacket); + mFinPackets.add(new FinAckPair(finPacket)); + } + + /** + * Attempt to ACK any FIN packets held by this conversation. + * @param ackPacket The ACK for a FIN previously added to this conversation. + */ + public void attemptAcknowledgementOfFin(PcapPacket ackPacket) { + // Precondition: verify that the packet pertains to this conversation. + onAddPrecondition(ackPacket); + // Mark unack'ed FIN(s) that this ACK matches as ACK'ed (there might be more than one in case of retransmissions..?) + mFinPackets.replaceAll(finAckPair -> !finAckPair.isAcknowledged() && finAckPair.isCorrespondingAckPacket(ackPacket) ? new FinAckPair(finAckPair.getFinPacket(), ackPacket) : finAckPair); + } + + /** + * Retrieves an unmodifiable view of the list of {@link FinAckPair}s associated with this {@code Conversation}. + * @return an unmodifiable view of the list of {@link FinAckPair}s associated with this {@code Conversation}. + */ + public List getFinAckPairs() { + return Collections.unmodifiableList(mFinPackets); + } + + /** + * Get if this {@code Conversation} is considered to have been gracefully shut down. + * A {@code Conversation} has been gracefully shut down if it contains a FIN+ACK pair for both directions + * (client to server, and server to client). + * @return {@code true} if the connection has been gracefully shut down, false otherwise. + */ + public boolean isGracefullyShutdown() { + // The conversation has been gracefully shut down if we have recorded a FIN from both the client and the server which have both been ack'ed. + return mFinPackets.stream().anyMatch(finAckPair -> finAckPair.isAcknowledged() && PcapPacketUtils.isSource(finAckPair.getFinPacket(), mClientIp, mClientPort)) && + mFinPackets.stream().anyMatch(finAckPair -> finAckPair.isAcknowledged() && PcapPacketUtils.isSource(finAckPair.getFinPacket(), mServerIp, mServerPort)); + } + + /** + * Add a TCP segment for which the RST flag is set to this {@code Conversation}. + * @param packet A {@link PcapPacket} wrapping a TCP segment pertaining to this {@code Conversation} for which the + * RST flag is set. + */ + public void addRstPacket(PcapPacket packet) { + /* + * TODO: + * When now also keeping track of RST packets, should we also...? + * 1) Prevent later packets from being added once a RST segment has been added? + * 2) Extend 'isGracefullyShutdown()' to also consider RST segments, or add another method, 'isShutdown()' that + * both considers FIN/ACK (graceful) as well as RST (abrupt/"ungraceful") shutdown? + * 3) Should it be impossible to associate more than one RST segment with each Conversation? + */ + onAddPrecondition(packet); + TcpPacket tcpPacket = packet.get(TcpPacket.class); + if (tcpPacket == null || !tcpPacket.getHeader().getRst()) { + throw new IllegalArgumentException("not a RST packet"); + } + mRstPackets.add(packet); + } + + /** + * Get the TCP segments pertaining to this {@code Conversation} for which it was detected that the RST flag is set. + * @return the TCP segments pertaining to this {@code Conversation} for which it was detected that the RST flag is + * set. + */ + public List getRstPackets() { + return Collections.unmodifiableList(mRstPackets); + } + + // ========================================================================================================= + // We simply reuse equals and hashCode methods of String.class to be able to use this class as a key + // in a Map. + + /** + * Note: currently, equality is determined based on pairwise equality of the elements of the four tuple + * ({@link #mClientIp}, {@link #mClientPort}, {@link #mServerIp}, {@link #mServerPort}) for {@code this} and + * {@code obj}. + * @param obj The object to test for equality with {@code this}. + * @return {@code true} if {@code obj} is considered equal to {@code this} based on the definition of equality given above. + */ + @Override + public boolean equals(Object obj) { + return obj instanceof Conversation && this.toString().equals(obj.toString()); + } + + @Override + public int hashCode() { + return toString().hashCode(); + } + // ========================================================================================================= + + @Override + public String toString() { + return String.format("%s:%d %s:%d", mClientIp, mClientPort, mServerIp, mServerPort); + } + + /** + * Invoke to verify that the precondition holds when a caller attempts to add a packet to this {@code Conversation}. + * An {@link IllegalArgumentException} is thrown if the precondition is violated. + * @param packet the packet to be added to this {@code Conversation} + */ + private void onAddPrecondition(PcapPacket packet) { + // Apply precondition to preserve class invariant: all packets in mPackets must match the 4 tuple that + // defines the conversation. + IpV4Packet ipPacket = Objects.requireNonNull(packet.get(IpV4Packet.class)); + // For now we only support TCP flows. + TcpPacket tcpPacket = Objects.requireNonNull(packet.get(TcpPacket.class)); + String ipSrc = ipPacket.getHeader().getSrcAddr().getHostAddress(); + String ipDst = ipPacket.getHeader().getDstAddr().getHostAddress(); + int srcPort = tcpPacket.getHeader().getSrcPort().valueAsInt(); + int dstPort = tcpPacket.getHeader().getDstPort().valueAsInt(); + String clientIp, serverIp; + int clientPort, serverPort; + if (ipSrc.equals(mClientIp)) { + clientIp = ipSrc; + clientPort = srcPort; + serverIp = ipDst; + serverPort = dstPort; + } else { + clientIp = ipDst; + clientPort = dstPort; + serverIp = ipSrc; + serverPort = srcPort; + } + if (!(clientIp.equals(mClientIp) && clientPort == mClientPort && + serverIp.equals(mServerIp) && serverPort == mServerPort)) { + throw new IllegalArgumentException( + String.format("Attempt to add packet that does not pertain to %s", + Conversation.class.getSimpleName())); + } + } + + /** + *

+ * Determines if the TCP packet contained in {@code packet} is a retransmission of a previously seen (logged) + * packet. + *

+ * + * + * TODO: + * the current implementation, which uses a set of previously seen sequence numbers, will consider a segment + * with a reused sequence number---occurring as a result of sequence number wrap around for a very long-lived + * connection---as a retransmission (and may therefore end up discarding it even though it is in fact NOT a + * retransmission). Ideas? + * + * + * @param packet The packet. + * @return {@code true} if {@code packet} was determined to be a retransmission, {@code false} otherwise. + */ + public boolean isRetransmission(PcapPacket packet) { + // Extract sequence number. + int seqNo = packet.get(TcpPacket.class).getHeader().getSequenceNumber(); + switch (getDirection(packet)) { + case CLIENT_TO_SERVER: + return mSeqNumbersClient.contains(seqNo); + case SERVER_TO_CLIENT: + return mSeqNumbersSrv.contains(seqNo); + default: + throw new AssertionError(String.format("Unexpected value of enum '%s'", + Direction.class.getSimpleName())); + } + } + + /** + *

+ * Is this {@code Conversation} a TLS session? + *

+ * + * Note: the current implementation simply examines the port number(s) for 443; it does not verify if the + * application data is indeed encrypted. + * + * @return {@code true} if this {@code Conversation} is interpreted as a TLS session, {@code false} otherwise. + */ + public boolean isTls() { + /* + * TODO: + * - may want to change this to be "return mServerPort == 443 || mClientPort == 443;" in order to also detect + * TLS in those cases where it is not possible to correctly label who is the client and who is the server, + * i.e., when the trace does not contain the SYN/SYNACK exchange. + * - current implementation relies on the server using the conventional TLS port number; may instead want to + * inspect the first 4 bytes of each potential TLS packet to see if they match the SSL record header. + * + * 08/31/18: Added unconvetional TLS ports used by WeMo plugs and LiFX bulb. + * 09/20/18: Moved hardcoded ports to other class to allow other classes to query the set of TLS ports. + */ + return TcpConversationUtils.isTlsPort(mServerPort); + } + + /** + * If this {@code Conversation} is backing a TLS session (i.e., if the value of {@link #isTls()} is {@code true}), + * get the packets labeled as TLS Application Data packets. This is a subset of the full set of payload-carrying + * packets (as returned by {@link #getPackets()}). An exception is thrown if this method is invoked on a + * {@code Conversation} for which {@link #isTls()} returns {@code false}. + * + * @return A list containing exactly those packets that could be identified as TLS Application Data packets (through + * inspecting of the SSL record header). The list may be empty, if no TLS application data packets have been + * recorded for this {@code Conversation}. + */ + public List getTlsApplicationDataPackets() { + if (!isTls()) { + throw new NoSuchElementException("cannot get TLS Application Data packets for non-TLS TCP conversation"); + } + return Collections.unmodifiableList(mTlsApplicationDataPackets); + } + + /** + * Extracts the TCP sequence number from {@code packet} and adds it to the proper set of sequence numbers by + * analyzing the direction of the packet. + * @param packet A TCP packet (wrapped in a {@code PcapPacket}) that was added to this conversation and whose + * sequence number is to be recorded as seen. + */ + private void addSeqNumber(PcapPacket packet) { + // Note: below check is redundant if client code is correct as the call to check the precondition should already + // have been made by the addXPacket method that invokes this method. As such, the call below may be removed in + // favor of speed, but the improvement will be minor, hence the added safety may be worth it. + onAddPrecondition(packet); + // Extract sequence number. + int seqNo = packet.get(TcpPacket.class).getHeader().getSequenceNumber(); + // Determine direction of packet and add packet's sequence number to corresponding set of sequence numbers. + switch (getDirection(packet)) { + case CLIENT_TO_SERVER: + // Client to server packet. + mSeqNumbersClient.add(seqNo); + break; + case SERVER_TO_CLIENT: + // Server to client packet. + mSeqNumbersSrv.add(seqNo); + break; + default: + throw new AssertionError(String.format("Unexpected value of enum '%s'", + Direction.class.getSimpleName())); + } + } + + /** + * Determine the direction of {@code packet}. An {@link IllegalArgumentException} is thrown if {@code packet} does + * not pertain to this conversation. + * + * @param packet The packet whose direction is to be determined. + * @return A {@link Direction} indicating the direction of the packet. + */ + public Direction getDirection(PcapPacket packet) { + IpV4Packet ipPacket = packet.get(IpV4Packet.class); + String ipSrc = ipPacket.getHeader().getSrcAddr().getHostAddress(); + String ipDst = ipPacket.getHeader().getDstAddr().getHostAddress(); + // Determine direction of packet. + if (ipSrc.equals(mClientIp) && ipDst.equals(mServerIp)) { + // Client to server packet. + return Direction.CLIENT_TO_SERVER; + } else if (ipSrc.equals(mServerIp) && ipDst.equals(mClientIp)) { + // Server to client packet. + return Direction.SERVER_TO_CLIENT; + } else { + throw new IllegalArgumentException("getDirection: packet not related to " + getClass().getSimpleName()); + } + } + + /** + * Utility enum for expressing the direction of a packet pertaining to this {@code Conversation}. + */ + public enum Direction { + + CLIENT_TO_SERVER { + @Override + public String toCompactString() { + return "*"; + } + }, + SERVER_TO_CLIENT { + @Override + public String toCompactString() { + return ""; + } + }; + + /** + * Get a compact string representation of this {@code Direction}. + * @return a compact string representation of this {@code Direction}. + */ + abstract public String toCompactString(); + + } + +} diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/FinAckPair.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/FinAckPair.java new file mode 100644 index 0000000..fe4f032 --- /dev/null +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/FinAckPair.java @@ -0,0 +1,134 @@ +package edu.uci.iotproject.trafficreassembly.layer3; + +import org.pcap4j.core.PcapPacket; +import org.pcap4j.packet.IpV4Packet; +import org.pcap4j.packet.TcpPacket; + +/** + * Groups a FIN packet and its corresponding ACK packet. Immutable and thread safe. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class FinAckPair { + + private final PcapPacket mFinPacket; + private final PcapPacket mCorrespondingAckPacket; + + /** + * Constructs a {@code FinAckPair} given a FIN packet. + * The corresponding ACK packet field is set to {@code null}. + * @param finPacket A FIN packet. + */ + public FinAckPair(PcapPacket finPacket) { + if (!finPacket.get(TcpPacket.class).getHeader().getFin()) { + throw new IllegalArgumentException("not a FIN packet"); + } + mFinPacket = finPacket; + mCorrespondingAckPacket = null; + } + + /** + * Constructs a {@code FinAckPair} given a FIN and an ACK packet. + * @param finPacket A FIN packet. + * @param correspondingAckPacket The ACK packet corresponding to {@code finPacket}. + */ + public FinAckPair(PcapPacket finPacket, PcapPacket correspondingAckPacket) { + // Enforce class invariant, i.e. that the FIN and ACK are related. + // Note that it is indirectly checked whether finPacket is indeed a FIN packet + // as isCorrespondingAckPacket calls the single parameter constructor. + if (!FinAckPair.isCorrespondingAckPacket(finPacket, correspondingAckPacket)) { + throw new IllegalArgumentException("FIN and ACK not related"); + } + mFinPacket = finPacket; + mCorrespondingAckPacket = correspondingAckPacket; + } + + /** + * Get the FIN packet of this pair. + * @return the FIN packet of this pair. + */ + public PcapPacket getFinPacket() { + return mFinPacket; + } + + /** + * Get the corresponding ACK packet of this pair, if any. + * @return the corresponding ACK packet of this pair, if any. + */ + public PcapPacket getCorrespondingAckPacket() { + return mCorrespondingAckPacket; + } + + /** + * Was the FIN in this {@code FinAckPair} acknowledged? + * + * @return {@code true} if the corresponding ACK has been set in this {@code FinAckPair}. + */ + public boolean isAcknowledged() { + return mFinPacket != null && mCorrespondingAckPacket != null; + } + + /** + * Checks if a given packet is an ACK corresponding to the FIN packet in this {@code FinAckPair}. + * @return {@code true} if {@code packet} is an ACK that corresponds to the FIN in this pair, {@code false} otherwise. + */ + public boolean isCorrespondingAckPacket(PcapPacket packet) { + IpV4Packet inputIpPacket = packet.get(IpV4Packet.class); + TcpPacket inputTcpPacket = packet.get(TcpPacket.class); + if (inputIpPacket == null || inputTcpPacket == null || !inputTcpPacket.getHeader().getAck()) { + return false; + } + + IpV4Packet finIpPacket = mFinPacket.get(IpV4Packet.class); + TcpPacket finTcpPacket = mFinPacket.get(TcpPacket.class); + + // Extract (srcIp:port,dstIp:port) for input and member (FIN) packets. + String inputPacketIpSrc = inputIpPacket.getHeader().getSrcAddr().getHostAddress(); + String inputPacketIpDst = inputIpPacket.getHeader().getDstAddr().getHostAddress(); + int inputPacketPortSrc = inputTcpPacket.getHeader().getSrcPort().valueAsInt(); + int inputPacketPortDst = inputTcpPacket.getHeader().getDstPort().valueAsInt(); + String finPacketIpSrc = finIpPacket.getHeader().getSrcAddr().getHostAddress(); + String finPacketIpDst = finIpPacket.getHeader().getDstAddr().getHostAddress(); + int finPacketPortSrc = finTcpPacket.getHeader().getSrcPort().valueAsInt(); + int finPacketPortDst = finTcpPacket.getHeader().getDstPort().valueAsInt(); + + // For the two packets to be related, the dst of one must be the src of the other. + // Split into multiple if statements for readability. First check IP fields, then ports. + if (!(inputPacketIpDst.equals(finPacketIpSrc) && finPacketIpDst.equals(inputPacketIpSrc))) { + return false; + } + if (!(inputPacketPortDst == finPacketPortSrc && finPacketPortDst == inputPacketPortSrc)) { + return false; + } + + // Packets are (most likely) related (part of same conversation/stream). + // Now all that is left for us to check is if the sequence numbers match. + // Note: recall that the FIN packet advances the seq numbers by 1, + // so the ACK number will be one larger than the seq. number in the FIN packet. + return inputTcpPacket.getHeader().getAcknowledgmentNumber() == finTcpPacket.getHeader().getSequenceNumber() + 1; + } + + /** + * Static method to check if two given packets are a FIN and the corresponding ACK packet. + * The purpose of this method is a workaround to enforce the class invariant in the two parameter constructor. + * Specifically, the following should be avoided: + * 
+     *     public FinAckPair(PcapPacket finPacket, PcapPacket correspondingAckPacket) {
+     *         mFinPacket = finPacket;
+     *         // Below line is considered bad practice as the object has not been fully initialized at this stage.
+     *         if (!this.isCorrespondingAckPacket(correspondingAckPacket)) {
+     *             // ... throw exception
+     *         }
+     *     }
+     *
+ * @param finPacket The FIN packet. + * @param ackPacket The ACK packet that is to be checked if it corresponds to the given FIN packet. + * @return {@code true} if the ACK corresponds to the FIN, {@code false} otherwise. + */ + private static boolean isCorrespondingAckPacket(PcapPacket finPacket, PcapPacket ackPacket) { + FinAckPair tmp = new FinAckPair(finPacket); + return tmp.isCorrespondingAckPacket(ackPacket); + } + +} diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/TcpReassembler.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/TcpReassembler.java new file mode 100644 index 0000000..e150875 --- /dev/null +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/TcpReassembler.java @@ -0,0 +1,260 @@ +package edu.uci.iotproject.trafficreassembly.layer3; + +import org.pcap4j.core.PacketListener; +import org.pcap4j.core.PcapPacket; +import org.pcap4j.packet.*; + +import java.util.*; + +/** + * Reassembles TCP conversations (streams). + * Note: current version only supports TCP over IPv4. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class TcpReassembler implements PacketListener { + + /** + * Holds open {@link Conversation}s, i.e., {@code Conversation}s that have not been detected as + * (gracefully) terminated based on the set of packets observed thus far. + * A {@link Conversation} is moved to {@link #mTerminatedConversations} if it can be determined that it is has + * terminated. Termination can be detected by a) observing two {@link FinAckPair}s, one in each direction, (graceful + * termination, see {@link Conversation#isGracefullyShutdown()}) or b) by observing a SYN packet that matches the + * four tuple of an existing {@code Conversation}, but which holds a different sequence number than the + * same-direction SYN packet recorded for the {@code Conversation}. + *

+ * Note that due to limitations of the {@link Set} interface (specifically, there is no {@code get(T t)} method), + * we have to resort to a {@link Map} (in which keys map to themselves) to "mimic" a set with {@code get(T t)} + * functionality. + * + * @see this question on StackOverflow.com + */ + private final Map mOpenConversations = new HashMap<>(); + + /** + * Holds terminated {@link Conversation}s. + */ + private final List mTerminatedConversations = new ArrayList<>(); + + @Override + public void gotPacket(PcapPacket pcapPacket) { + IpV4Packet ipPacket = pcapPacket.get(IpV4Packet.class); + TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class); + + if (ipPacket == null || tcpPacket == null) { + return; + } + // ... TODO? + processPacket(pcapPacket); +// Class clazz = pcapPacket.getClass(); +// RadiotapPacket radiotapPacket = pcapPacket.get(RadiotapPacket.class); +// Dot11ManagementPacket dot11ManagementPacket = pcapPacket.get(Dot11ManagementPacket.class); +// if (dot11ManagementPacket != null) { +// return; +// } +// if (radiotapPacket != null) { +// processRadiotapPacket(pcapPacket); +// } + } + + /** + * Get the reassembled TCP connections. Note that if this is called while packets are still being processed (by + * calls to {@link #gotPacket(PcapPacket)}), the behavior is undefined and the returned list may be inconsistent. + * @return The reassembled TCP connections. + */ + public List getTcpConversations() { + ArrayList combined = new ArrayList<>(); + combined.addAll(mTerminatedConversations); + combined.addAll(mOpenConversations.values()); + return combined; + } + + private void processRadiotapPacket(PcapPacket pcapPacket) { + RadiotapPacket radiotapPacket = pcapPacket.get(RadiotapPacket.class); + + RadiotapPacket.RadiotapHeader header = radiotapPacket.getHeader(); + short length = header.getLength(); + ArrayList radiotapData = header.getDataFields(); + // TODO: We can handle this 802.11 QoS data by creating our own class + // TODO: We only need to handle the first few bytes for source, destination, receiver, and transmitter + // TODO: addresses + Packet dataPacket = radiotapPacket.getPayload(); + int dataLength = dataPacket.length(); + } + + private void processPacket(PcapPacket pcapPacket) { + TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class); + // Handle client connection initiation attempts. + if (tcpPacket.getHeader().getSyn() && !tcpPacket.getHeader().getAck()) { + // A segment with the SYN flag set, but no ACK flag indicates that a client is attempting to initiate a new + // connection. + processNewConnectionRequest(pcapPacket); + return; + } + // Handle server connection initiation acknowledgement + if (tcpPacket.getHeader().getSyn() && tcpPacket.getHeader().getAck()) { + // A segment with both the SYN and ACK flags set indicates that the server has accepted the client's request + // to initiate a new connection. + processNewConnectionAck(pcapPacket); + return; + } + // Handle resets + if (tcpPacket.getHeader().getRst()) { + processRstPacket(pcapPacket); + return; + } + // Handle FINs + if (tcpPacket.getHeader().getFin()) { + // Handle FIN packet. + processFinPacket(pcapPacket); + } + // Handle ACKs (currently only ACKs of FINS) + if (tcpPacket.getHeader().getAck()) { + processAck(pcapPacket); + } + // Handle packets that carry payload (application data). + if (tcpPacket.getPayload() != null) { + processPayloadPacket(pcapPacket); + } + } + + private void processNewConnectionRequest(PcapPacket clientSynPacket) { + // A SYN w/o ACK always originates from the client. + Conversation conv = Conversation.fromPcapPacket(clientSynPacket, true); + conv.addSynPacket(clientSynPacket); + // Is there an ongoing conversation for the same four tuple (clientIp, clientPort, serverIp, serverPort) as + // found in the new SYN packet? + Conversation ongoingConv = mOpenConversations.get(conv); + if (ongoingConv != null) { + if (ongoingConv.isRetransmission(clientSynPacket)) { + // SYN retransmission detected, do nothing. + return; + // TODO: the way retransmission detection is implemented may cause a bug for connections where we have + // not recorded the initial SYN, but only the SYN ACK, as retransmission is determined by comparing the + // sequence numbers of initial SYNs -- and if no initial SYN is present for the Conversation, the new + // SYN will be interpreted as a retransmission. Possible fix: let isRentransmission ALWAYS return false + // when presented with a SYN packet when the Conversation already holds a SYN ACK packet? + } else { + // New SYN has different sequence number than SYN recorded for ongoingConv, so this must be an attempt + // to establish a new conversation with the same four tuple as ongoingConv. + // Mark existing connection as terminated. + // TODO: is this 100% theoretically correct, e.g., if many connection attempts are made back to back? And RST packets? + mTerminatedConversations.add(ongoingConv); + mOpenConversations.remove(ongoingConv); + } + } + // Finally, update the map of open connections with the new connection. + mOpenConversations.put(conv, conv); + } + + + /* + * TODO a problem across the board for all processXPacket methods below: + * if we start the capture in the middle of a TCP connection, we will not have an entry for the conversation in the + * map as we have not seen the initial SYN packet. + * Two ways we can address this: + * a) Perform null-checks and ignore packets for which we have not seen SYN + * + easy to get correct + * - we discard data (issue for long-lived connections!) + * b) Add a corresponding conversation entry whenever we encounter a packet that does not map to a conversation + * + we consider all data + * - not immediately clear if this will introduce bugs (incorrectly mapping packets to wrong conversations?) + * + * [[[ I went with option b) for now; see getOngoingConversationOrCreateNew(PcapPacket pcapPacket). ]]] + */ + + private void processNewConnectionAck(PcapPacket srvSynPacket) { + // Find the corresponding ongoing connection, if any (if we start the capture just *after* the initial SYN, no + // ongoing conversation entry will exist, so it must be created in that case). +// Conversation conv = mOpenConversations.get(Conversation.fromPcapPacket(srvSynPacket, false)); + Conversation conv = getOngoingConversationOrCreateNew(srvSynPacket); + // Note: exploits &&'s short-circuit operation: only attempts to add non-retransmissions. + if (!conv.isRetransmission(srvSynPacket) && !conv.addSynPacket(srvSynPacket)) { + // For safety/debugging: if NOT a retransmission and add fails, + // something has gone terribly wrong/invariant is broken. +// throw new AssertionError("Attempt to add SYN ACK packet that was NOT a retransmission failed." + +// Conversation.class.getSimpleName() + " invariant broken."); + } + } + + private void processRstPacket(PcapPacket rstPacket) { + Conversation conv = getOngoingConversationOrCreateNew(rstPacket); + // Add RST packet to conversation. + conv.addRstPacket(rstPacket); + // Move conversation to set of terminated conversations. + mTerminatedConversations.add(conv); + mOpenConversations.remove(conv, conv); + } + + private void processFinPacket(PcapPacket finPacket) { +// getOngoingConversationForPacket(finPacket).addFinPacket(finPacket); + getOngoingConversationOrCreateNew(finPacket).addFinPacket(finPacket); + } + + private void processAck(PcapPacket ackPacket) { +// getOngoingConversationForPacket(ackPacket).attemptAcknowledgementOfFin(ackPacket); + // Note that unlike the style for SYN, FIN, and payload packets, for "ACK only" packets, we want to avoid + // creating a new conversation. + Conversation conv = getOngoingConversationForPacket(ackPacket); + if (conv != null) { + // The ACK may be an ACK of a FIN, so attempt to mark the FIN as ack'ed. + conv.attemptAcknowledgementOfFin(ackPacket); + if (conv.isGracefullyShutdown()) { + // Move conversation to set of terminated conversations. + mTerminatedConversations.add(conv); + mOpenConversations.remove(conv); + } + } + // Note: add (additional) processing of ACKs (that are not ACKs of FINs) as necessary here... + } + + private void processPayloadPacket(PcapPacket pcapPacket) { +// getOngoingConversationForPacket(pcapPacket).addPacket(pcapPacket, true); + getOngoingConversationOrCreateNew(pcapPacket).addPacket(pcapPacket, true); + } + + /** + * Locates an ongoing conversation (if any) that {@code pcapPacket} pertains to. + * @param pcapPacket The packet that is to be mapped to an ongoing {@code Conversation}. + * @return The {@code Conversation} matching {@code pcapPacket} or {@code null} if there is no match. + */ + private Conversation getOngoingConversationForPacket(PcapPacket pcapPacket) { + // We cannot know if this is a client-to-server or server-to-client packet without trying both options... + Conversation conv = mOpenConversations.get(Conversation.fromPcapPacket(pcapPacket, true)); + if (conv == null) { + conv = mOpenConversations.get(Conversation.fromPcapPacket(pcapPacket, false)); + } + return conv; + } + + /** + * Like {@link #getOngoingConversationForPacket(PcapPacket)}, but creates and inserts a new {@code Conversation} + * into {@link #mOpenConversations} if no open conversation is found (i.e., in the case that + * {@link #getOngoingConversationForPacket(PcapPacket)} returns {@code null}). + * + * @param pcapPacket The packet that is to be mapped to an ongoing {@code Conversation}. + * @return The existing, ongoing {@code Conversation} matching {@code pcapPacket} or the newly created one in case + * no match was found. + */ + private Conversation getOngoingConversationOrCreateNew(PcapPacket pcapPacket) { + Conversation conv = getOngoingConversationForPacket(pcapPacket); + if (conv == null) { + TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class); + if (tcpPacket.getHeader().getSyn() && tcpPacket.getHeader().getAck()) { + // A SYN ACK packet always originates from the server (it is a reply to the initial SYN packet from the client) + conv = Conversation.fromPcapPacket(pcapPacket, false); + } else { + // TODO: can we do anything else but arbitrarily select who is designated as the server in this case? + // We can check if the IP prefix matches a local IP when handling traffic observed inside the local + // network, but that obviously won't be a useful strategy for an observer at the WAN port. + String srcIp = pcapPacket.get(IpV4Packet.class).getHeader().getSrcAddr().getHostAddress(); + // TODO: REPLACE THE ROUTER'S IP WITH A PARAMETER!!! + boolean clientIsSrc = srcIp.startsWith("10.") || srcIp.startsWith("192.168.") || srcIp.equals("128.195.205.105"); + conv = Conversation.fromPcapPacket(pcapPacket, clientIsSrc); + } + mOpenConversations.put(conv, conv); + } + return conv; + } +} diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java index cc958ee..f07072a 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java @@ -1,6 +1,6 @@ package edu.uci.iotproject.util; -import edu.uci.iotproject.Conversation; +import edu.uci.iotproject.trafficreassembly.layer3.Conversation; import edu.uci.iotproject.analysis.PcapPacketPair; import edu.uci.iotproject.analysis.TcpConversationUtils; import edu.uci.iotproject.analysis.TriggerTrafficExtractor;