From: rtrimana Date: Mon, 14 Jan 2019 18:03:15 +0000 (-0800) Subject: Merge branch 'master' of https://github.uci.edu/rtrimana/smart_home_traffic X-Git-Url: http://plrg.eecs.uci.edu/git/?p=pingpong.git;a=commitdiff_plain;h=5f2d56277c6be7130fb2f8faf9b2706d3e861265;hp=c6e0577434a49ac80de864c3b70d997a413411e8 Merge branch 'master' of https://github.uci.edu/rtrimana/smart_home_traffic --- diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Conversation.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Conversation.java deleted file mode 100644 index 490dc4f..0000000 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Conversation.java +++ /dev/null @@ -1,585 +0,0 @@ -package edu.uci.iotproject; - -import edu.uci.iotproject.analysis.TcpConversationUtils; -import edu.uci.iotproject.util.PcapPacketUtils; -import org.pcap4j.core.PcapPacket; -import org.pcap4j.packet.IpV4Packet; -import org.pcap4j.packet.Packet; -import org.pcap4j.packet.TcpPacket; - -import java.util.*; - -/** - * Models a (TCP) conversation/connection/session/flow (packet's belonging to the same session between a client and a - * server). - * Holds a list of {@link PcapPacket}s identified as pertaining to the flow. Note that this list is not - * considered when determining equality of two {@code Conversation} instances in order to allow for a - * {@code Conversation} to function as a key in data structures such as {@link java.util.Map} and {@link java.util.Set}. - * See {@link #equals(Object)} for the definition of equality. - * - * @author Janus Varmarken {@literal } - * @author Rahmadi Trimananda {@literal } - */ -public class Conversation { - - /* Begin instance properties */ - /** - * The IP of the host that is considered the client (i.e. the host that initiates the conversation) - * in this conversation. - */ - private final String mClientIp; - - /** - * The port number used by the host that is considered the client in this conversation. - */ - private final int mClientPort; - - /** - * The IP of the host that is considered the server (i.e. is the responder) in this conversation. - */ - private final String mServerIp; - - /** - * The port number used by the server in this conversation. - */ - private final int mServerPort; - - /** - * The list of packets (with payload) pertaining to this conversation. - */ - private final List mPackets; - - /** - * If {@link #isTls()} is {@code true}, this list contains the subset of {@link #mPackets} which are TLS Application - * Data packets. - */ - private final List mTlsApplicationDataPackets; - - /** - * Contains the sequence numbers used thus far by the host that is considered the client in this - * {@code Conversation}. - * Used for filtering out retransmissions. - */ - private final Set mSeqNumbersClient; - - /** - * Contains the sequence numbers used thus far by the host that is considered the server in this - * {@code Conversation}. - * Used for filtering out retransmissions. - */ - private final Set mSeqNumbersSrv; - - /** - * List of SYN packets pertaining to this conversation. - */ - private final List mSynPackets; - - /** - * List of pairs FINs and their corresponding ACKs associated with this conversation. - */ - private final List mFinPackets; - - /** - * List of RST packets associated with this conversation. - */ - private final List mRstPackets; - - /** - * Boolean to mark the packet as Application Data based on the previous packet that reaches MTU - */ - private boolean mApplicationData; - /* End instance properties */ - - /** - * Factory method for creating a {@code Conversation} from a {@link PcapPacket}. - * @param pcapPacket The {@code PcapPacket} that wraps a TCP segment for which a {@code Conversation} is to be initiated. - * @param clientIsSrc If {@code true}, the source address and source port found in the IP datagram and TCP segment - * wrapped in the {@code PcapPacket} are regarded as pertaining to the client, and the destination - * address and destination port are regarded as pertaining to the server---and vice versa if set - * to {@code false}. - * @return A {@code Conversation} initiated with ip:port for client and server according to the direction of the packet. - */ - public static Conversation fromPcapPacket(PcapPacket pcapPacket, boolean clientIsSrc) { - IpV4Packet ipPacket = pcapPacket.get(IpV4Packet.class); - TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class); - String clientIp = clientIsSrc ? ipPacket.getHeader().getSrcAddr().getHostAddress() : - ipPacket.getHeader().getDstAddr().getHostAddress(); - String srvIp = clientIsSrc ? ipPacket.getHeader().getDstAddr().getHostAddress() : - ipPacket.getHeader().getSrcAddr().getHostAddress(); - int clientPort = clientIsSrc ? tcpPacket.getHeader().getSrcPort().valueAsInt() : - tcpPacket.getHeader().getDstPort().valueAsInt(); - int srvPort = clientIsSrc ? tcpPacket.getHeader().getDstPort().valueAsInt() : - tcpPacket.getHeader().getSrcPort().valueAsInt(); - return new Conversation(clientIp, clientPort, srvIp, srvPort); - } - - /** - * Constructs a new {@code Conversation}. - * @param clientIp The IP of the host that is considered the client (i.e. the host that initiates the conversation) - * in the conversation. - * @param clientPort The port number used by the client for the conversation. - * @param serverIp The IP of the host that is considered the server (i.e. is the responder) in the conversation. - * @param serverPort The port number used by the server for the conversation. - */ - public Conversation(String clientIp, int clientPort, String serverIp, int serverPort) { - this.mClientIp = clientIp; - this.mClientPort = clientPort; - this.mServerIp = serverIp; - this.mServerPort = serverPort; - this.mPackets = new ArrayList<>(); - this.mTlsApplicationDataPackets = new ArrayList<>(); - this.mSeqNumbersClient = new HashSet<>(); - this.mSeqNumbersSrv = new HashSet<>(); - this.mSynPackets = new ArrayList<>(); - this.mFinPackets = new ArrayList<>(); - this.mRstPackets = new ArrayList<>(); - this.mApplicationData = false; - } - - /** - * Add a packet to the list of packets associated with this conversation. - * @param packet The packet that is to be added to (associated with) this conversation. - * @param ignoreRetransmissions Boolean value indicating if retransmissions should be ignored. - * If set to {@code true}, {@code packet} will not be added to the - * internal list of packets pertaining to this {@code Conversation} - * iff the sequence number of {@code packet} was already - * seen in a previous packet. - */ - public void addPacket(PcapPacket packet, boolean ignoreRetransmissions) { - // Precondition: verify that packet does indeed pertain to conversation. - onAddPrecondition(packet); - if (ignoreRetransmissions && isRetransmission(packet)) { - // Packet is a retransmission. Ignore it. - return; - } - // Select direction-dependent set of sequence numbers seen so far and update it with sequence number of new packet. - addSeqNumber(packet); - // Finally add packet to list of packets pertaining to this conversation. - mPackets.add(packet); - // Preserve order of packets in list: sort according to timestamp. - if (mPackets.size() > 1 && - mPackets.get(mPackets.size()-1).getTimestamp().isBefore(mPackets.get(mPackets.size()-2).getTimestamp())) { - Collections.sort(mPackets, (o1, o2) -> { - if (o1.getTimestamp().isBefore(o2.getTimestamp())) { return -1; } - else if (o2.getTimestamp().isBefore(o1.getTimestamp())) { return 1; } - else { return 0; } - }); - } - // If TLS, inspect packet to see if it's a TLS Application Data packet, and if so add it to the list of TLS - // Application Data packets. - if (isTls()) { - TcpPacket tcpPacket = packet.get(TcpPacket.class); - Packet tcpPayload = tcpPacket.getPayload(); - if (tcpPayload == null) { - return; - } - byte[] rawPayload = tcpPayload.getRawData(); - // The SSL record header is at the front of the payload and is 5 bytes long. - // The SSL record header type field (the first byte) is set to 23 if it is an Application Data packet. - if (rawPayload != null && rawPayload.length >= 5) { - if (rawPayload[0] == 23) { - mTlsApplicationDataPackets.add(packet); - // Consider the following packet a data packet if this packet's size == MTU size 1448 - if (rawPayload.length >= 1448) - mApplicationData = true; - } else if (rawPayload[0] == 20) { - // Do nothing for now - CHANGE_CIPHER_SPEC - } else if (rawPayload[0] == 21) { - // Do nothing for now - ALERT - } else if (rawPayload[0] == 22) { - // Do nothing for now - HANDSHAKE - } else { - // If it is TLS with payload, but rawPayload[0] != 23 - if (mApplicationData == true) { - // It is a continuation of the previous packet if the previous packet reaches MTU size 1448 and - // it is not either type 20, 21, or 22 - mTlsApplicationDataPackets.add(packet); - if (rawPayload.length < 1448) - mApplicationData = false; - } - } - } - } - } - - /** - * Get a list of packets pertaining to this {@code Conversation}. - * The returned list is a read-only list. - * @return the list of packets pertaining to this {@code Conversation}. - */ - public List getPackets() { - // Return read-only view to prevent external code from manipulating internal state (preserve invariant). - return Collections.unmodifiableList(mPackets); - } - - /** - * Records a TCP SYN packet as pertaining to this conversation (adds it to the the internal list). - * Attempts to add duplicate SYN packets will be ignored, and the caller is made aware of the attempt to add a - * duplicate by the return value being {@code false}. - * - * @param synPacket A {@link PcapPacket} wrapping a TCP SYN packet. - * @return {@code true} if the packet was successfully added to this {@code Conversation}, {@code false} otherwise. - */ - public boolean addSynPacket(PcapPacket synPacket) { - onAddPrecondition(synPacket); - final IpV4Packet synPacketIpSection = synPacket.get(IpV4Packet.class); - final TcpPacket synPacketTcpSection = synPacket.get(TcpPacket.class); - if (synPacketTcpSection == null || !synPacketTcpSection.getHeader().getSyn()) { - throw new IllegalArgumentException("Not a SYN packet."); - } - // We are only interested in recording one copy of the two SYN packets (one SYN packet in each direction), i.e., - // we want to discard retransmitted SYN packets. - if (mSynPackets.size() >= 2) { - return false; - } - // Check the set of recorded SYN packets to see if we have already recorded a SYN packet going in the same - // direction as the packet given in the argument. - boolean matchingPrevSyn = mSynPackets.stream().anyMatch(p -> { - IpV4Packet pIp = p.get(IpV4Packet.class); - TcpPacket pTcp = p.get(TcpPacket.class); - boolean srcAddrMatch = synPacketIpSection.getHeader().getSrcAddr().getHostAddress(). - equals(pIp.getHeader().getSrcAddr().getHostAddress()); - boolean dstAddrMatch = synPacketIpSection.getHeader().getDstAddr().getHostAddress(). - equals(pIp.getHeader().getDstAddr().getHostAddress()); - boolean srcPortMatch = synPacketTcpSection.getHeader().getSrcPort().valueAsInt() == - pTcp.getHeader().getSrcPort().valueAsInt(); - boolean dstPortMatch = synPacketTcpSection.getHeader().getDstPort().valueAsInt() == - pTcp.getHeader().getDstPort().valueAsInt(); - return srcAddrMatch && dstAddrMatch && srcPortMatch && dstPortMatch; - }); - if (matchingPrevSyn) { - return false; - } - // Update direction-dependent set of sequence numbers and record/log packet. - addSeqNumber(synPacket); - return mSynPackets.add(synPacket); - - /* - mSynPackets.stream().anyMatch(p -> { - IpV4Packet pIp = p.get(IpV4Packet.class); - TcpPacket pTcp = p.get(TcpPacket.class); - boolean srcAddrMatch = synPacketIpSection.getHeader().getSrcAddr().getHostAddress(). - equals(pIp.getHeader().getSrcAddr().getHostAddress()); - boolean dstAddrMatch = synPacketIpSection.getHeader().getDstAddr().getHostAddress(). - equals(pIp.getHeader().getDstAddr().getHostAddress()); - boolean srcPortMatch = synPacketTcpSection.getHeader().getSrcPort().valueAsInt() == - pTcp.getHeader().getSrcPort().valueAsInt(); - boolean dstPortMatch = synPacketTcpSection.getHeader().getDstPort().value() == - pTcp.getHeader().getDstPort().value(); - - boolean fourTupleMatch = srcAddrMatch && dstAddrMatch && srcPortMatch && dstPortMatch; - - boolean seqNoMatch = synPacketTcpSection.getHeader().getSequenceNumber() == - pTcp.getHeader().getSequenceNumber(); - - if (fourTupleMatch && !seqNoMatch) { - // If the four tuple that identifies the conversation matches, but the sequence number is different, - // it means that this SYN packet is, in fact, an attempt to establish a **new** connection, and hence - // the given packet is NOT part of this conversation, even though the ip:port combinations are (by - // chance) selected such that they match this conversation. - throw new IllegalArgumentException("Attempt to add SYN packet that belongs to a different conversation " + - "(which is identified by the same four tuple as this conversation)"); - } - return fourTupleMatch && seqNoMatch; - }); - */ - } - - /** - * Get a list of SYN packets pertaining to this {@code Conversation}. - * The returned list is a read-only list. - * @return the list of SYN packets pertaining to this {@code Conversation}. - */ - public List getSynPackets() { - return Collections.unmodifiableList(mSynPackets); - } - - /** - * Adds a TCP FIN packet to the list of TCP FIN packets associated with this conversation. - * @param finPacket The TCP FIN packet that is to be added to (associated with) this conversation. - */ - public void addFinPacket(PcapPacket finPacket) { - // Precondition: verify that packet does indeed pertain to conversation. - onAddPrecondition(finPacket); - // TODO: should call addSeqNumber here? - addSeqNumber(finPacket); - mFinPackets.add(new FinAckPair(finPacket)); - } - - /** - * Attempt to ACK any FIN packets held by this conversation. - * @param ackPacket The ACK for a FIN previously added to this conversation. - */ - public void attemptAcknowledgementOfFin(PcapPacket ackPacket) { - // Precondition: verify that the packet pertains to this conversation. - onAddPrecondition(ackPacket); - // Mark unack'ed FIN(s) that this ACK matches as ACK'ed (there might be more than one in case of retransmissions..?) - mFinPackets.replaceAll(finAckPair -> !finAckPair.isAcknowledged() && finAckPair.isCorrespondingAckPacket(ackPacket) ? new FinAckPair(finAckPair.getFinPacket(), ackPacket) : finAckPair); - } - - /** - * Retrieves an unmodifiable view of the list of {@link FinAckPair}s associated with this {@code Conversation}. - * @return an unmodifiable view of the list of {@link FinAckPair}s associated with this {@code Conversation}. - */ - public List getFinAckPairs() { - return Collections.unmodifiableList(mFinPackets); - } - - /** - * Get if this {@code Conversation} is considered to have been gracefully shut down. - * A {@code Conversation} has been gracefully shut down if it contains a FIN+ACK pair for both directions - * (client to server, and server to client). - * @return {@code true} if the connection has been gracefully shut down, false otherwise. - */ - public boolean isGracefullyShutdown() { - // The conversation has been gracefully shut down if we have recorded a FIN from both the client and the server which have both been ack'ed. - return mFinPackets.stream().anyMatch(finAckPair -> finAckPair.isAcknowledged() && PcapPacketUtils.isSource(finAckPair.getFinPacket(), mClientIp, mClientPort)) && - mFinPackets.stream().anyMatch(finAckPair -> finAckPair.isAcknowledged() && PcapPacketUtils.isSource(finAckPair.getFinPacket(), mServerIp, mServerPort)); - } - - /** - * Add a TCP segment for which the RST flag is set to this {@code Conversation}. - * @param packet A {@link PcapPacket} wrapping a TCP segment pertaining to this {@code Conversation} for which the - * RST flag is set. - */ - public void addRstPacket(PcapPacket packet) { - /* - * TODO: - * When now also keeping track of RST packets, should we also...? - * 1) Prevent later packets from being added once a RST segment has been added? - * 2) Extend 'isGracefullyShutdown()' to also consider RST segments, or add another method, 'isShutdown()' that - * both considers FIN/ACK (graceful) as well as RST (abrupt/"ungraceful") shutdown? - * 3) Should it be impossible to associate more than one RST segment with each Conversation? - */ - onAddPrecondition(packet); - TcpPacket tcpPacket = packet.get(TcpPacket.class); - if (tcpPacket == null || !tcpPacket.getHeader().getRst()) { - throw new IllegalArgumentException("not a RST packet"); - } - mRstPackets.add(packet); - } - - /** - * Get the TCP segments pertaining to this {@code Conversation} for which it was detected that the RST flag is set. - * @return the TCP segments pertaining to this {@code Conversation} for which it was detected that the RST flag is - * set. - */ - public List getRstPackets() { - return Collections.unmodifiableList(mRstPackets); - } - - // ========================================================================================================= - // We simply reuse equals and hashCode methods of String.class to be able to use this class as a key - // in a Map. - - /** - * Note: currently, equality is determined based on pairwise equality of the elements of the four tuple - * ({@link #mClientIp}, {@link #mClientPort}, {@link #mServerIp}, {@link #mServerPort}) for {@code this} and - * {@code obj}. - * @param obj The object to test for equality with {@code this}. - * @return {@code true} if {@code obj} is considered equal to {@code this} based on the definition of equality given above. - */ - @Override - public boolean equals(Object obj) { - return obj instanceof Conversation && this.toString().equals(obj.toString()); - } - - @Override - public int hashCode() { - return toString().hashCode(); - } - // ========================================================================================================= - - @Override - public String toString() { - return String.format("%s:%d %s:%d", mClientIp, mClientPort, mServerIp, mServerPort); - } - - /** - * Invoke to verify that the precondition holds when a caller attempts to add a packet to this {@code Conversation}. - * An {@link IllegalArgumentException} is thrown if the precondition is violated. - * @param packet the packet to be added to this {@code Conversation} - */ - private void onAddPrecondition(PcapPacket packet) { - // Apply precondition to preserve class invariant: all packets in mPackets must match the 4 tuple that - // defines the conversation. - IpV4Packet ipPacket = Objects.requireNonNull(packet.get(IpV4Packet.class)); - // For now we only support TCP flows. - TcpPacket tcpPacket = Objects.requireNonNull(packet.get(TcpPacket.class)); - String ipSrc = ipPacket.getHeader().getSrcAddr().getHostAddress(); - String ipDst = ipPacket.getHeader().getDstAddr().getHostAddress(); - int srcPort = tcpPacket.getHeader().getSrcPort().valueAsInt(); - int dstPort = tcpPacket.getHeader().getDstPort().valueAsInt(); - String clientIp, serverIp; - int clientPort, serverPort; - if (ipSrc.equals(mClientIp)) { - clientIp = ipSrc; - clientPort = srcPort; - serverIp = ipDst; - serverPort = dstPort; - } else { - clientIp = ipDst; - clientPort = dstPort; - serverIp = ipSrc; - serverPort = srcPort; - } - if (!(clientIp.equals(mClientIp) && clientPort == mClientPort && - serverIp.equals(mServerIp) && serverPort == mServerPort)) { - throw new IllegalArgumentException( - String.format("Attempt to add packet that does not pertain to %s", - Conversation.class.getSimpleName())); - } - } - - /** - *

- * Determines if the TCP packet contained in {@code packet} is a retransmission of a previously seen (logged) - * packet. - *

- * - * - * TODO: - * the current implementation, which uses a set of previously seen sequence numbers, will consider a segment - * with a reused sequence number---occurring as a result of sequence number wrap around for a very long-lived - * connection---as a retransmission (and may therefore end up discarding it even though it is in fact NOT a - * retransmission). Ideas? - * - * - * @param packet The packet. - * @return {@code true} if {@code packet} was determined to be a retransmission, {@code false} otherwise. - */ - public boolean isRetransmission(PcapPacket packet) { - // Extract sequence number. - int seqNo = packet.get(TcpPacket.class).getHeader().getSequenceNumber(); - switch (getDirection(packet)) { - case CLIENT_TO_SERVER: - return mSeqNumbersClient.contains(seqNo); - case SERVER_TO_CLIENT: - return mSeqNumbersSrv.contains(seqNo); - default: - throw new AssertionError(String.format("Unexpected value of enum '%s'", - Direction.class.getSimpleName())); - } - } - - /** - *

- * Is this {@code Conversation} a TLS session? - *

- * - * Note: the current implementation simply examines the port number(s) for 443; it does not verify if the - * application data is indeed encrypted. - * - * @return {@code true} if this {@code Conversation} is interpreted as a TLS session, {@code false} otherwise. - */ - public boolean isTls() { - /* - * TODO: - * - may want to change this to be "return mServerPort == 443 || mClientPort == 443;" in order to also detect - * TLS in those cases where it is not possible to correctly label who is the client and who is the server, - * i.e., when the trace does not contain the SYN/SYNACK exchange. - * - current implementation relies on the server using the conventional TLS port number; may instead want to - * inspect the first 4 bytes of each potential TLS packet to see if they match the SSL record header. - * - * 08/31/18: Added unconvetional TLS ports used by WeMo plugs and LiFX bulb. - * 09/20/18: Moved hardcoded ports to other class to allow other classes to query the set of TLS ports. - */ - return TcpConversationUtils.isTlsPort(mServerPort); - } - - /** - * If this {@code Conversation} is backing a TLS session (i.e., if the value of {@link #isTls()} is {@code true}), - * get the packets labeled as TLS Application Data packets. This is a subset of the full set of payload-carrying - * packets (as returned by {@link #getPackets()}). An exception is thrown if this method is invoked on a - * {@code Conversation} for which {@link #isTls()} returns {@code false}. - * - * @return A list containing exactly those packets that could be identified as TLS Application Data packets (through - * inspecting of the SSL record header). The list may be empty, if no TLS application data packets have been - * recorded for this {@code Conversation}. - */ - public List getTlsApplicationDataPackets() { - if (!isTls()) { - throw new NoSuchElementException("cannot get TLS Application Data packets for non-TLS TCP conversation"); - } - return Collections.unmodifiableList(mTlsApplicationDataPackets); - } - - /** - * Extracts the TCP sequence number from {@code packet} and adds it to the proper set of sequence numbers by - * analyzing the direction of the packet. - * @param packet A TCP packet (wrapped in a {@code PcapPacket}) that was added to this conversation and whose - * sequence number is to be recorded as seen. - */ - private void addSeqNumber(PcapPacket packet) { - // Note: below check is redundant if client code is correct as the call to check the precondition should already - // have been made by the addXPacket method that invokes this method. As such, the call below may be removed in - // favor of speed, but the improvement will be minor, hence the added safety may be worth it. - onAddPrecondition(packet); - // Extract sequence number. - int seqNo = packet.get(TcpPacket.class).getHeader().getSequenceNumber(); - // Determine direction of packet and add packet's sequence number to corresponding set of sequence numbers. - switch (getDirection(packet)) { - case CLIENT_TO_SERVER: - // Client to server packet. - mSeqNumbersClient.add(seqNo); - break; - case SERVER_TO_CLIENT: - // Server to client packet. - mSeqNumbersSrv.add(seqNo); - break; - default: - throw new AssertionError(String.format("Unexpected value of enum '%s'", - Direction.class.getSimpleName())); - } - } - - /** - * Determine the direction of {@code packet}. An {@link IllegalArgumentException} is thrown if {@code packet} does - * not pertain to this conversation. - * - * @param packet The packet whose direction is to be determined. - * @return A {@link Direction} indicating the direction of the packet. - */ - public Direction getDirection(PcapPacket packet) { - IpV4Packet ipPacket = packet.get(IpV4Packet.class); - String ipSrc = ipPacket.getHeader().getSrcAddr().getHostAddress(); - String ipDst = ipPacket.getHeader().getDstAddr().getHostAddress(); - // Determine direction of packet. - if (ipSrc.equals(mClientIp) && ipDst.equals(mServerIp)) { - // Client to server packet. - return Direction.CLIENT_TO_SERVER; - } else if (ipSrc.equals(mServerIp) && ipDst.equals(mClientIp)) { - // Server to client packet. - return Direction.SERVER_TO_CLIENT; - } else { - throw new IllegalArgumentException("getDirection: packet not related to " + getClass().getSimpleName()); - } - } - - /** - * Utility enum for expressing the direction of a packet pertaining to this {@code Conversation}. - */ - public enum Direction { - - CLIENT_TO_SERVER { - @Override - public String toCompactString() { - return "*"; - } - }, - SERVER_TO_CLIENT { - @Override - public String toCompactString() { - return ""; - } - }; - - /** - * Get a compact string representation of this {@code Direction}. - * @return a compact string representation of this {@code Direction}. - */ - abstract public String toCompactString(); - - } - -} diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FinAckPair.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FinAckPair.java deleted file mode 100644 index d4451f3..0000000 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FinAckPair.java +++ /dev/null @@ -1,134 +0,0 @@ -package edu.uci.iotproject; - -import org.pcap4j.core.PcapPacket; -import org.pcap4j.packet.IpV4Packet; -import org.pcap4j.packet.TcpPacket; - -/** - * Groups a FIN packet and its corresponding ACK packet. Immutable and thread safe. - * - * @author Janus Varmarken {@literal } - * @author Rahmadi Trimananda {@literal } - */ -public class FinAckPair { - - private final PcapPacket mFinPacket; - private final PcapPacket mCorrespondingAckPacket; - - /** - * Constructs a {@code FinAckPair} given a FIN packet. - * The corresponding ACK packet field is set to {@code null}. - * @param finPacket A FIN packet. - */ - public FinAckPair(PcapPacket finPacket) { - if (!finPacket.get(TcpPacket.class).getHeader().getFin()) { - throw new IllegalArgumentException("not a FIN packet"); - } - mFinPacket = finPacket; - mCorrespondingAckPacket = null; - } - - /** - * Constructs a {@code FinAckPair} given a FIN and an ACK packet. - * @param finPacket A FIN packet. - * @param correspondingAckPacket The ACK packet corresponding to {@code finPacket}. - */ - public FinAckPair(PcapPacket finPacket, PcapPacket correspondingAckPacket) { - // Enforce class invariant, i.e. that the FIN and ACK are related. - // Note that it is indirectly checked whether finPacket is indeed a FIN packet - // as isCorrespondingAckPacket calls the single parameter constructor. - if (!FinAckPair.isCorrespondingAckPacket(finPacket, correspondingAckPacket)) { - throw new IllegalArgumentException("FIN and ACK not related"); - } - mFinPacket = finPacket; - mCorrespondingAckPacket = correspondingAckPacket; - } - - /** - * Get the FIN packet of this pair. - * @return the FIN packet of this pair. - */ - public PcapPacket getFinPacket() { - return mFinPacket; - } - - /** - * Get the corresponding ACK packet of this pair, if any. - * @return the corresponding ACK packet of this pair, if any. - */ - public PcapPacket getCorrespondingAckPacket() { - return mCorrespondingAckPacket; - } - - /** - * Was the FIN in this {@code FinAckPair} acknowledged? - * - * @return {@code true} if the corresponding ACK has been set in this {@code FinAckPair}. - */ - public boolean isAcknowledged() { - return mFinPacket != null && mCorrespondingAckPacket != null; - } - - /** - * Checks if a given packet is an ACK corresponding to the FIN packet in this {@code FinAckPair}. - * @return {@code true} if {@code packet} is an ACK that corresponds to the FIN in this pair, {@code false} otherwise. - */ - public boolean isCorrespondingAckPacket(PcapPacket packet) { - IpV4Packet inputIpPacket = packet.get(IpV4Packet.class); - TcpPacket inputTcpPacket = packet.get(TcpPacket.class); - if (inputIpPacket == null || inputTcpPacket == null || !inputTcpPacket.getHeader().getAck()) { - return false; - } - - IpV4Packet finIpPacket = mFinPacket.get(IpV4Packet.class); - TcpPacket finTcpPacket = mFinPacket.get(TcpPacket.class); - - // Extract (srcIp:port,dstIp:port) for input and member (FIN) packets. - String inputPacketIpSrc = inputIpPacket.getHeader().getSrcAddr().getHostAddress(); - String inputPacketIpDst = inputIpPacket.getHeader().getDstAddr().getHostAddress(); - int inputPacketPortSrc = inputTcpPacket.getHeader().getSrcPort().valueAsInt(); - int inputPacketPortDst = inputTcpPacket.getHeader().getDstPort().valueAsInt(); - String finPacketIpSrc = finIpPacket.getHeader().getSrcAddr().getHostAddress(); - String finPacketIpDst = finIpPacket.getHeader().getDstAddr().getHostAddress(); - int finPacketPortSrc = finTcpPacket.getHeader().getSrcPort().valueAsInt(); - int finPacketPortDst = finTcpPacket.getHeader().getDstPort().valueAsInt(); - - // For the two packets to be related, the dst of one must be the src of the other. - // Split into multiple if statements for readability. First check IP fields, then ports. - if (!(inputPacketIpDst.equals(finPacketIpSrc) && finPacketIpDst.equals(inputPacketIpSrc))) { - return false; - } - if (!(inputPacketPortDst == finPacketPortSrc && finPacketPortDst == inputPacketPortSrc)) { - return false; - } - - // Packets are (most likely) related (part of same conversation/stream). - // Now all that is left for us to check is if the sequence numbers match. - // Note: recall that the FIN packet advances the seq numbers by 1, - // so the ACK number will be one larger than the seq. number in the FIN packet. - return inputTcpPacket.getHeader().getAcknowledgmentNumber() == finTcpPacket.getHeader().getSequenceNumber() + 1; - } - - /** - * Static method to check if two given packets are a FIN and the corresponding ACK packet. - * The purpose of this method is a workaround to enforce the class invariant in the two parameter constructor. - * Specifically, the following should be avoided: - * 
-     *     public FinAckPair(PcapPacket finPacket, PcapPacket correspondingAckPacket) {
-     *         mFinPacket = finPacket;
-     *         // Below line is considered bad practice as the object has not been fully initialized at this stage.
-     *         if (!this.isCorrespondingAckPacket(correspondingAckPacket)) {
-     *             // ... throw exception
-     *         }
-     *     }
-     *
- * @param finPacket The FIN packet. - * @param ackPacket The ACK packet that is to be checked if it corresponds to the given FIN packet. - * @return {@code true} if the ACK corresponds to the FIN, {@code false} otherwise. - */ - private static boolean isCorrespondingAckPacket(PcapPacket finPacket, PcapPacket ackPacket) { - FinAckPair tmp = new FinAckPair(finPacket); - return tmp.isCorrespondingAckPacket(ackPacket); - } - -} diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FlowPatternFinder.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FlowPatternFinder.java index 1d0a9ba..c384852 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FlowPatternFinder.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FlowPatternFinder.java @@ -3,6 +3,7 @@ package edu.uci.iotproject; import edu.uci.iotproject.comparison.ComparisonFunctions; import edu.uci.iotproject.comparison.CompleteMatchPatternComparisonResult; import edu.uci.iotproject.comparison.PatternComparisonTask; +import edu.uci.iotproject.trafficreassembly.layer3.Conversation; import org.pcap4j.core.NotOpenException; import org.pcap4j.core.PcapHandle; import org.pcap4j.core.PcapNativeException; @@ -15,7 +16,6 @@ import java.io.*; import java.net.UnknownHostException; import java.text.DateFormat; import java.text.SimpleDateFormat; -import java.time.temporal.ChronoField; import java.util.*; import java.util.concurrent.*; diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java index 5b184d2..81eb3b0 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java @@ -3,10 +3,9 @@ package edu.uci.iotproject; import static edu.uci.iotproject.analysis.UserAction.Type; import edu.uci.iotproject.analysis.*; -import edu.uci.iotproject.comparison.seqalignment.ExtractedSequence; -import edu.uci.iotproject.comparison.seqalignment.SequenceAlignment; -import edu.uci.iotproject.comparison.seqalignment.SequenceExtraction; import edu.uci.iotproject.io.TriggerTimesFileReader; +import edu.uci.iotproject.trafficreassembly.layer3.Conversation; +import edu.uci.iotproject.trafficreassembly.layer3.TcpReassembler; import edu.uci.iotproject.util.PcapPacketUtils; import edu.uci.iotproject.util.PrintUtils; import org.apache.commons.math3.stat.clustering.Cluster; @@ -15,8 +14,6 @@ import org.pcap4j.core.*; import org.pcap4j.packet.namednumber.DataLinkType; import java.io.EOFException; -import java.io.File; -import java.io.PrintWriter; import java.net.UnknownHostException; import java.time.Instant; import java.util.*; diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/TcpReassembler.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/TcpReassembler.java deleted file mode 100644 index 72f0c1c..0000000 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/TcpReassembler.java +++ /dev/null @@ -1,260 +0,0 @@ -package edu.uci.iotproject; - -import org.pcap4j.core.PacketListener; -import org.pcap4j.core.PcapPacket; -import org.pcap4j.packet.*; - -import java.util.*; - -/** - * Reassembles TCP conversations (streams). - * Note: current version only supports TCP over IPv4. - * - * @author Janus Varmarken {@literal } - * @author Rahmadi Trimananda {@literal } - */ -public class TcpReassembler implements PacketListener { - - /** - * Holds open {@link Conversation}s, i.e., {@code Conversation}s that have not been detected as - * (gracefully) terminated based on the set of packets observed thus far. - * A {@link Conversation} is moved to {@link #mTerminatedConversations} if it can be determined that it is has - * terminated. Termination can be detected by a) observing two {@link FinAckPair}s, one in each direction, (graceful - * termination, see {@link Conversation#isGracefullyShutdown()}) or b) by observing a SYN packet that matches the - * four tuple of an existing {@code Conversation}, but which holds a different sequence number than the - * same-direction SYN packet recorded for the {@code Conversation}. - *

- * Note that due to limitations of the {@link Set} interface (specifically, there is no {@code get(T t)} method), - * we have to resort to a {@link Map} (in which keys map to themselves) to "mimic" a set with {@code get(T t)} - * functionality. - * - * @see this question on StackOverflow.com - */ - private final Map mOpenConversations = new HashMap<>(); - - /** - * Holds terminated {@link Conversation}s. - */ - private final List mTerminatedConversations = new ArrayList<>(); - - @Override - public void gotPacket(PcapPacket pcapPacket) { - IpV4Packet ipPacket = pcapPacket.get(IpV4Packet.class); - TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class); - - if (ipPacket == null || tcpPacket == null) { - return; - } - // ... TODO? - processPacket(pcapPacket); -// Class clazz = pcapPacket.getClass(); -// RadiotapPacket radiotapPacket = pcapPacket.get(RadiotapPacket.class); -// Dot11ManagementPacket dot11ManagementPacket = pcapPacket.get(Dot11ManagementPacket.class); -// if (dot11ManagementPacket != null) { -// return; -// } -// if (radiotapPacket != null) { -// processRadiotapPacket(pcapPacket); -// } - } - - /** - * Get the reassembled TCP connections. Note that if this is called while packets are still being processed (by - * calls to {@link #gotPacket(PcapPacket)}), the behavior is undefined and the returned list may be inconsistent. - * @return The reassembled TCP connections. - */ - public List getTcpConversations() { - ArrayList combined = new ArrayList<>(); - combined.addAll(mTerminatedConversations); - combined.addAll(mOpenConversations.values()); - return combined; - } - - private void processRadiotapPacket(PcapPacket pcapPacket) { - RadiotapPacket radiotapPacket = pcapPacket.get(RadiotapPacket.class); - - RadiotapPacket.RadiotapHeader header = radiotapPacket.getHeader(); - short length = header.getLength(); - ArrayList radiotapData = header.getDataFields(); - // TODO: We can handle this 802.11 QoS data by creating our own class - // TODO: We only need to handle the first few bytes for source, destination, receiver, and transmitter - // TODO: addresses - Packet dataPacket = radiotapPacket.getPayload(); - int dataLength = dataPacket.length(); - } - - private void processPacket(PcapPacket pcapPacket) { - TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class); - // Handle client connection initiation attempts. - if (tcpPacket.getHeader().getSyn() && !tcpPacket.getHeader().getAck()) { - // A segment with the SYN flag set, but no ACK flag indicates that a client is attempting to initiate a new - // connection. - processNewConnectionRequest(pcapPacket); - return; - } - // Handle server connection initiation acknowledgement - if (tcpPacket.getHeader().getSyn() && tcpPacket.getHeader().getAck()) { - // A segment with both the SYN and ACK flags set indicates that the server has accepted the client's request - // to initiate a new connection. - processNewConnectionAck(pcapPacket); - return; - } - // Handle resets - if (tcpPacket.getHeader().getRst()) { - processRstPacket(pcapPacket); - return; - } - // Handle FINs - if (tcpPacket.getHeader().getFin()) { - // Handle FIN packet. - processFinPacket(pcapPacket); - } - // Handle ACKs (currently only ACKs of FINS) - if (tcpPacket.getHeader().getAck()) { - processAck(pcapPacket); - } - // Handle packets that carry payload (application data). - if (tcpPacket.getPayload() != null) { - processPayloadPacket(pcapPacket); - } - } - - private void processNewConnectionRequest(PcapPacket clientSynPacket) { - // A SYN w/o ACK always originates from the client. - Conversation conv = Conversation.fromPcapPacket(clientSynPacket, true); - conv.addSynPacket(clientSynPacket); - // Is there an ongoing conversation for the same four tuple (clientIp, clientPort, serverIp, serverPort) as - // found in the new SYN packet? - Conversation ongoingConv = mOpenConversations.get(conv); - if (ongoingConv != null) { - if (ongoingConv.isRetransmission(clientSynPacket)) { - // SYN retransmission detected, do nothing. - return; - // TODO: the way retransmission detection is implemented may cause a bug for connections where we have - // not recorded the initial SYN, but only the SYN ACK, as retransmission is determined by comparing the - // sequence numbers of initial SYNs -- and if no initial SYN is present for the Conversation, the new - // SYN will be interpreted as a retransmission. Possible fix: let isRentransmission ALWAYS return false - // when presented with a SYN packet when the Conversation already holds a SYN ACK packet? - } else { - // New SYN has different sequence number than SYN recorded for ongoingConv, so this must be an attempt - // to establish a new conversation with the same four tuple as ongoingConv. - // Mark existing connection as terminated. - // TODO: is this 100% theoretically correct, e.g., if many connection attempts are made back to back? And RST packets? - mTerminatedConversations.add(ongoingConv); - mOpenConversations.remove(ongoingConv); - } - } - // Finally, update the map of open connections with the new connection. - mOpenConversations.put(conv, conv); - } - - - /* - * TODO a problem across the board for all processXPacket methods below: - * if we start the capture in the middle of a TCP connection, we will not have an entry for the conversation in the - * map as we have not seen the initial SYN packet. - * Two ways we can address this: - * a) Perform null-checks and ignore packets for which we have not seen SYN - * + easy to get correct - * - we discard data (issue for long-lived connections!) - * b) Add a corresponding conversation entry whenever we encounter a packet that does not map to a conversation - * + we consider all data - * - not immediately clear if this will introduce bugs (incorrectly mapping packets to wrong conversations?) - * - * [[[ I went with option b) for now; see getOngoingConversationOrCreateNew(PcapPacket pcapPacket). ]]] - */ - - private void processNewConnectionAck(PcapPacket srvSynPacket) { - // Find the corresponding ongoing connection, if any (if we start the capture just *after* the initial SYN, no - // ongoing conversation entry will exist, so it must be created in that case). -// Conversation conv = mOpenConversations.get(Conversation.fromPcapPacket(srvSynPacket, false)); - Conversation conv = getOngoingConversationOrCreateNew(srvSynPacket); - // Note: exploits &&'s short-circuit operation: only attempts to add non-retransmissions. - if (!conv.isRetransmission(srvSynPacket) && !conv.addSynPacket(srvSynPacket)) { - // For safety/debugging: if NOT a retransmission and add fails, - // something has gone terribly wrong/invariant is broken. -// throw new AssertionError("Attempt to add SYN ACK packet that was NOT a retransmission failed." + -// Conversation.class.getSimpleName() + " invariant broken."); - } - } - - private void processRstPacket(PcapPacket rstPacket) { - Conversation conv = getOngoingConversationOrCreateNew(rstPacket); - // Add RST packet to conversation. - conv.addRstPacket(rstPacket); - // Move conversation to set of terminated conversations. - mTerminatedConversations.add(conv); - mOpenConversations.remove(conv, conv); - } - - private void processFinPacket(PcapPacket finPacket) { -// getOngoingConversationForPacket(finPacket).addFinPacket(finPacket); - getOngoingConversationOrCreateNew(finPacket).addFinPacket(finPacket); - } - - private void processAck(PcapPacket ackPacket) { -// getOngoingConversationForPacket(ackPacket).attemptAcknowledgementOfFin(ackPacket); - // Note that unlike the style for SYN, FIN, and payload packets, for "ACK only" packets, we want to avoid - // creating a new conversation. - Conversation conv = getOngoingConversationForPacket(ackPacket); - if (conv != null) { - // The ACK may be an ACK of a FIN, so attempt to mark the FIN as ack'ed. - conv.attemptAcknowledgementOfFin(ackPacket); - if (conv.isGracefullyShutdown()) { - // Move conversation to set of terminated conversations. - mTerminatedConversations.add(conv); - mOpenConversations.remove(conv); - } - } - // Note: add (additional) processing of ACKs (that are not ACKs of FINs) as necessary here... - } - - private void processPayloadPacket(PcapPacket pcapPacket) { -// getOngoingConversationForPacket(pcapPacket).addPacket(pcapPacket, true); - getOngoingConversationOrCreateNew(pcapPacket).addPacket(pcapPacket, true); - } - - /** - * Locates an ongoing conversation (if any) that {@code pcapPacket} pertains to. - * @param pcapPacket The packet that is to be mapped to an ongoing {@code Conversation}. - * @return The {@code Conversation} matching {@code pcapPacket} or {@code null} if there is no match. - */ - private Conversation getOngoingConversationForPacket(PcapPacket pcapPacket) { - // We cannot know if this is a client-to-server or server-to-client packet without trying both options... - Conversation conv = mOpenConversations.get(Conversation.fromPcapPacket(pcapPacket, true)); - if (conv == null) { - conv = mOpenConversations.get(Conversation.fromPcapPacket(pcapPacket, false)); - } - return conv; - } - - /** - * Like {@link #getOngoingConversationForPacket(PcapPacket)}, but creates and inserts a new {@code Conversation} - * into {@link #mOpenConversations} if no open conversation is found (i.e., in the case that - * {@link #getOngoingConversationForPacket(PcapPacket)} returns {@code null}). - * - * @param pcapPacket The packet that is to be mapped to an ongoing {@code Conversation}. - * @return The existing, ongoing {@code Conversation} matching {@code pcapPacket} or the newly created one in case - * no match was found. - */ - private Conversation getOngoingConversationOrCreateNew(PcapPacket pcapPacket) { - Conversation conv = getOngoingConversationForPacket(pcapPacket); - if (conv == null) { - TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class); - if (tcpPacket.getHeader().getSyn() && tcpPacket.getHeader().getAck()) { - // A SYN ACK packet always originates from the server (it is a reply to the initial SYN packet from the client) - conv = Conversation.fromPcapPacket(pcapPacket, false); - } else { - // TODO: can we do anything else but arbitrarily select who is designated as the server in this case? - // We can check if the IP prefix matches a local IP when handling traffic observed inside the local - // network, but that obviously won't be a useful strategy for an observer at the WAN port. - String srcIp = pcapPacket.get(IpV4Packet.class).getHeader().getSrcAddr().getHostAddress(); - // TODO: REPLACE THE ROUTER'S IP WITH A PARAMETER!!! - boolean clientIsSrc = srcIp.startsWith("10.") || srcIp.startsWith("192.168.") || srcIp.equals("128.195.205.105"); - conv = Conversation.fromPcapPacket(pcapPacket, clientIsSrc); - } - mOpenConversations.put(conv, conv); - } - return conv; - } -} diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TcpConversationUtils.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TcpConversationUtils.java index 7484d8a..f1c264b 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TcpConversationUtils.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TcpConversationUtils.java @@ -1,6 +1,6 @@ package edu.uci.iotproject.analysis; -import edu.uci.iotproject.Conversation; +import edu.uci.iotproject.trafficreassembly.layer3.Conversation; import edu.uci.iotproject.DnsMap; import edu.uci.iotproject.util.PcapPacketUtils; import org.pcap4j.core.PcapPacket; diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TrafficLabeler.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TrafficLabeler.java index ebc87b1..983de12 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TrafficLabeler.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/TrafficLabeler.java @@ -1,8 +1,8 @@ package edu.uci.iotproject.analysis; -import edu.uci.iotproject.Conversation; +import edu.uci.iotproject.trafficreassembly.layer3.Conversation; import edu.uci.iotproject.DnsMap; -import edu.uci.iotproject.TcpReassembler; +import edu.uci.iotproject.trafficreassembly.layer3.TcpReassembler; import org.pcap4j.core.PacketListener; import org.pcap4j.core.PcapPacket; diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/AbstractPatternComparisonResult.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/AbstractPatternComparisonResult.java index f3bd585..cadadbe 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/AbstractPatternComparisonResult.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/AbstractPatternComparisonResult.java @@ -1,6 +1,6 @@ package edu.uci.iotproject.comparison; -import edu.uci.iotproject.Conversation; +import edu.uci.iotproject.trafficreassembly.layer3.Conversation; import edu.uci.iotproject.FlowPattern; /** diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/ComparisonFunctions.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/ComparisonFunctions.java index 15eda20..960d6a7 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/ComparisonFunctions.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/ComparisonFunctions.java @@ -1,6 +1,6 @@ package edu.uci.iotproject.comparison; -import edu.uci.iotproject.Conversation; +import edu.uci.iotproject.trafficreassembly.layer3.Conversation; import edu.uci.iotproject.FlowPattern; import org.pcap4j.core.PcapPacket; import org.pcap4j.packet.TcpPacket; diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/CompleteMatchPatternComparisonResult.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/CompleteMatchPatternComparisonResult.java index 3ddd884..960a0d0 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/CompleteMatchPatternComparisonResult.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/CompleteMatchPatternComparisonResult.java @@ -1,6 +1,6 @@ package edu.uci.iotproject.comparison; -import edu.uci.iotproject.Conversation; +import edu.uci.iotproject.trafficreassembly.layer3.Conversation; import edu.uci.iotproject.FlowPattern; /** diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/PatternComparisonTask.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/PatternComparisonTask.java index 808dee7..5a888b7 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/PatternComparisonTask.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/PatternComparisonTask.java @@ -1,6 +1,6 @@ package edu.uci.iotproject.comparison; -import edu.uci.iotproject.Conversation; +import edu.uci.iotproject.trafficreassembly.layer3.Conversation; import edu.uci.iotproject.FlowPattern; import java.util.concurrent.Callable; diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/seqalignment/ExtractedSequence.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/seqalignment/ExtractedSequence.java index 297107d..2d193a9 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/seqalignment/ExtractedSequence.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/seqalignment/ExtractedSequence.java @@ -1,6 +1,6 @@ package edu.uci.iotproject.comparison.seqalignment; -import edu.uci.iotproject.Conversation; +import edu.uci.iotproject.trafficreassembly.layer3.Conversation; import org.pcap4j.core.PcapPacket; import java.util.List; diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/seqalignment/SequenceExtraction.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/seqalignment/SequenceExtraction.java index c611968..6aaa318 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/seqalignment/SequenceExtraction.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/seqalignment/SequenceExtraction.java @@ -1,9 +1,8 @@ package edu.uci.iotproject.comparison.seqalignment; -import edu.uci.iotproject.Conversation; +import edu.uci.iotproject.trafficreassembly.layer3.Conversation; import edu.uci.iotproject.analysis.TcpConversationUtils; -import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.stream.Collectors; diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/AbstractClusterMatcher.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/AbstractClusterMatcher.java new file mode 100644 index 0000000..bda6493 --- /dev/null +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/AbstractClusterMatcher.java @@ -0,0 +1,49 @@ +package edu.uci.iotproject.detection; + +import org.pcap4j.core.PcapPacket; + +import java.util.List; +import java.util.Objects; + +/** + * Base class for classes that search a traffic trace for sequences of packets that "belong to" a given cluster (in + * other words, classes that attempt to classify traffic as pertaining to a given cluster). + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +abstract public class AbstractClusterMatcher { + + /** + * The cluster that describes the sequence of packets that this {@link AbstractClusterMatcher} is trying to detect + * in the observed traffic. + */ + protected final List> mCluster; + + protected AbstractClusterMatcher(List> cluster) { + // ===================== PRECONDITION SECTION ===================== + cluster = Objects.requireNonNull(cluster, "cluster cannot be null"); + if (cluster.isEmpty() || cluster.stream().anyMatch(inner -> inner.isEmpty())) { + throw new IllegalArgumentException("cluster is empty (or contains an empty inner List)"); + } + for (List clusterMember : cluster) { + if (clusterMember.size() != cluster.get(0).size()) { + throw new IllegalArgumentException("All sequences in cluster must contain the same number of packets"); + } + } + // ================================================================ + // Let the subclass prune the provided cluster + mCluster = pruneCluster(cluster); + } + + /** + * Allows subclasses to specify how to prune the input cluster provided to the constructor. + * @param cluster The input cluster provided to the constructor. + * @return The pruned cluster to use in place of the input cluster. + */ + abstract protected List> pruneCluster(List> cluster); + + // TODO: move Direction outside Conversation so that this is less confusing. +// abstract protected Conversation.Direction[] getPacketDirections(List packets); + +} diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/ClusterMatcher.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/ClusterMatcher.java deleted file mode 100644 index 33d6dbb..0000000 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/ClusterMatcher.java +++ /dev/null @@ -1,362 +0,0 @@ -package edu.uci.iotproject.detection; - -import edu.uci.iotproject.Conversation; -import edu.uci.iotproject.TcpReassembler; -import edu.uci.iotproject.analysis.TcpConversationUtils; -import edu.uci.iotproject.io.PcapHandleReader; -import edu.uci.iotproject.util.PrintUtils; -import org.pcap4j.core.*; - -import java.time.ZoneId; -import java.util.*; -import java.util.stream.Collectors; - -import static edu.uci.iotproject.util.PcapPacketUtils.*; - -/** - * Searches a traffic trace for sequences of packets "belong to" a given cluster (in other words, attempts to classify - * traffic as pertaining to a given cluster). - * - * @author Janus Varmarken {@literal } - * @author Rahmadi Trimananda {@literal } - */ -public class ClusterMatcher implements PacketListener { - - // Test client - public static void main(String[] args) throws PcapNativeException, NotOpenException { - -// String path = "/scratch/July-2018"; // Rahmadi - String path = "/Users/varmarken/temp/UCI IoT Project/experiments"; // Janus - final String inputPcapFile = path + "/2018-07/dlink/dlink.wlan1.local.pcap"; - final String signatureFile = path + "/2018-07/dlink/offSignature1.sig"; - - List> signature = PrintUtils.deserializeClustersFromFile(signatureFile); - ClusterMatcher clusterMatcher = new ClusterMatcher(signature, null, - (sig, match) -> System.out.println( - String.format("[ !!! SIGNATURE DETECTED AT %s !!! ]", - match.get(0).getTimestamp().atZone(ZoneId.of("America/Los_Angeles"))) - ) - ); - - PcapHandle handle; - try { - handle = Pcaps.openOffline(inputPcapFile, PcapHandle.TimestampPrecision.NANO); - } catch (PcapNativeException pne) { - handle = Pcaps.openOffline(inputPcapFile); - } - PcapHandleReader reader = new PcapHandleReader(handle, p -> true, clusterMatcher); - reader.readFromHandle(); - clusterMatcher.performDetection(); - } - - /** - * The cluster that describes the sequence of packets that this {@link ClusterMatcher} is trying to detect in the - * observed traffic. - */ - private final List> mCluster; - - /** - * The ordered directions of packets in the sequences that make up {@link #mCluster}. - */ - private final Conversation.Direction[] mClusterMemberDirections; - - /** - * For reassembling the observed traffic into TCP connections. - */ - private final TcpReassembler mTcpReassembler = new TcpReassembler(); - - /** - * IP of the router's WAN port (if analyzed traffic is captured at the ISP's point of view). - */ - private final String mRouterWanIp; - - private final ClusterMatchObserver[] mObservers; - - /** - * Create a {@link ClusterMatcher}. - * @param cluster The cluster that traffic is matched against. - * @param routerWanIp The router's WAN IP if examining traffic captured at the ISP's point of view (used for - * determining the direction of packets). - * @param detectionObservers Client code that wants to get notified whenever the {@link ClusterMatcher} detects that - * (a subset of) the examined traffic is similar to the traffic that makes up - * {@code cluster}, i.e., when the examined traffic is classified as pertaining to - * {@code cluster}. - */ - public ClusterMatcher(List> cluster, String routerWanIp, ClusterMatchObserver... detectionObservers) { - // ===================== PRECONDITION SECTION ===================== - cluster = Objects.requireNonNull(cluster, "cluster cannot be null"); - if (cluster.isEmpty() || cluster.stream().anyMatch(inner -> inner.isEmpty())) { - throw new IllegalArgumentException("cluster is empty (or contains an empty inner List)"); - } - mObservers = Objects.requireNonNull(detectionObservers, "detectionObservers cannot be null"); - if (mObservers.length == 0) { - throw new IllegalArgumentException("no detectionObservers provided"); - } - // Build the cluster members' direction sequence. - // Note: assumes that the provided cluster was captured within the local network (routerWanIp is set to null). - mClusterMemberDirections = getPacketDirections(cluster.get(0), null); - /* - * Enforce restriction on cluster members: all representatives must exhibit the same direction pattern and - * contain the same number of packets. Note that this is a somewhat heavy operation, so it may be disabled later - * on in favor of performance. However, it is only run once (at instantiation), so the overhead may be warranted - * in order to ensure correctness, especially during the development/debugging phase. - */ - if (cluster.stream(). - anyMatch(inner -> !Arrays.equals(mClusterMemberDirections, getPacketDirections(inner, null)))) { - throw new IllegalArgumentException( - "cluster members must contain the same number of packets and exhibit the same packet direction " + - "pattern" - ); - } - // ================================================================ - // Prune the provided cluster. - mCluster = pruneCluster(cluster); - mRouterWanIp = routerWanIp; - } - - @Override - public void gotPacket(PcapPacket packet) { - // Present packet to TCP reassembler so that it can be mapped to a connection (if it is a TCP packet). - mTcpReassembler.gotPacket(packet); - } - - /** - * Get the cluster that describes the packet sequence that this {@link ClusterMatcher} is searching for. - * @return the cluster that describes the packet sequence that this {@link ClusterMatcher} is searching for. - */ - public List> getCluster() { - return mCluster; - } - - public void performDetection() { - /* - * Let's start out simple by building a version that only works for signatures that do not span across multiple - * TCP conversations... - */ - for (Conversation c : mTcpReassembler.getTcpConversations()) { - if (c.isTls() && c.getTlsApplicationDataPackets().isEmpty() || !c.isTls() && c.getPackets().isEmpty()) { - // Skip empty conversations. - continue; - } - for (List signatureSequence : mCluster) { - if (isTlsSequence(signatureSequence) != c.isTls()) { - // We consider it a mismatch if one is a TLS application data sequence and the other is not. - continue; - } - // Fetch set of packets to examine based on TLS or not. - List cPkts = c.isTls() ? c.getTlsApplicationDataPackets() : c.getPackets(); - /* - * Note: we embed the attempt to detect the signature sequence in a loop in order to capture those cases - * where the same signature sequence appears multiple times in one Conversation. - * - * Note: since we expect all sequences that together make up the signature to exhibit the same direction - * pattern, we can simply pass the precomputed direction array for the signature sequence so that it - * won't have to be recomputed internally in each call to findSubsequenceInSequence(). - */ - Optional> match; - while ((match = findSubsequenceInSequence(signatureSequence, cPkts, mClusterMemberDirections, null)). - isPresent()) { - List matchSeq = match.get(); - // Notify observers about the match. - Arrays.stream(mObservers).forEach(o -> o.onMatch(ClusterMatcher.this, matchSeq)); - /* - * Get the index in cPkts of the last packet in the sequence of packets that matches the searched - * signature sequence. - */ - int matchSeqEndIdx = cPkts.indexOf(matchSeq.get(matchSeq.size()-1)); - // We restart the search for the signature sequence immediately after that index, so truncate cPkts. - cPkts = cPkts.stream().skip(matchSeqEndIdx + 1).collect(Collectors.toList()); - } - } - /* - * TODO: - * if no item in cluster matches, also perform a distance-based matching to cover those cases where we did - * not manage to capture every single mutation of the sequence during training. - * - * Need to compute average/centroid of cluster to do so...? Compute within-cluster variance, then check if - * distance between input conversation and cluster average/centroid is smaller than or equal to the computed - * variance? - */ - } - } - - /** - * Checks if {@code sequence} is a sequence of TLS packets. Note: the current implementation relies on inspection - * of the port numbers when deciding between TLS vs. non-TLS. Therefore, only the first packet of {@code sequence} - * is examined as it is assumed that all packets in {@code sequence} pertain to the same {@link Conversation} and - * hence share the same set of two src/dst port numbers (albeit possibly alternating between which one is the src - * and which one is the dst, as packets in {@code sequence} may be in alternating directions). - * @param sequence The sequence of packets for which it is to be determined if it is a sequence of TLS packets or - * non-TLS packets. - * @return {@code true} if {@code sequence} is a sequence of TLS packets, {@code false} otherwise. - */ - private boolean isTlsSequence(List sequence) { - // NOTE: Assumes ALL packets in sequence pertain to the same TCP connection! - PcapPacket firstPkt = sequence.get(0); - int srcPort = getSourcePort(firstPkt); - int dstPort = getDestinationPort(firstPkt); - return TcpConversationUtils.isTlsPort(srcPort) || TcpConversationUtils.isTlsPort(dstPort); - } - - /** - * Examine if a given sequence of packets ({@code sequence}) contains a given shorter sequence of packets - * ({@code subsequence}). Note: the current implementation actually searches for a substring as it does not allow - * for interleaving packets in {@code sequence} that are not in {@code subsequence}; for example, if - * {@code subsequence} consists of packet lengths [2, 3, 5] and {@code sequence} consists of packet lengths - * [2, 3, 4, 5], the result will be that there is no match (because of the interleaving 4). If we are to allow - * interleaving packets, we need a modified version of - * this. - * - * @param subsequence The sequence to search for. - * @param sequence The sequence to search. - * @param subsequenceDirections The directions of packets in {@code subsequence} such that for all {@code i}, - * {@code subsequenceDirections[i]} is the direction of the packet returned by - * {@code subsequence.get(i)}. May be set to {@code null}, in which this call will - * internally compute the packet directions. - * @param sequenceDirections The directions of packets in {@code sequence} such that for all {@code i}, - * {@code sequenceDirections[i]} is the direction of the packet returned by - * {@code sequence.get(i)}. May be set to {@code null}, in which this call will internally - * compute the packet directions. - * - * @return An {@link Optional} containing the part of {@code sequence} that matches {@code subsequence}, or an empty - * {@link Optional} if no part of {@code sequence} matches {@code subsequence}. - */ - private Optional> findSubsequenceInSequence(List subsequence, - List sequence, - Conversation.Direction[] subsequenceDirections, - Conversation.Direction[] sequenceDirections) { - if (sequence.size() < subsequence.size()) { - // If subsequence is longer, it cannot be contained in sequence. - return Optional.empty(); - } - if (isTlsSequence(subsequence) != isTlsSequence(sequence)) { - // We consider it a mismatch if one is a TLS application data sequence and the other is not. - return Optional.empty(); - } - // If packet directions have not been precomputed by calling code, we need to construct them. - if (subsequenceDirections == null) { - subsequenceDirections = getPacketDirections(subsequence, mRouterWanIp); - } - if (sequenceDirections == null) { - sequenceDirections = getPacketDirections(sequence, mRouterWanIp); - } - int subseqIdx = 0; - int seqIdx = 0; - while (seqIdx < sequence.size()) { - PcapPacket subseqPkt = subsequence.get(subseqIdx); - PcapPacket seqPkt = sequence.get(seqIdx); - // We only have a match if packet lengths and directions match. - if (subseqPkt.getOriginalLength() == seqPkt.getOriginalLength() && - subsequenceDirections[subseqIdx] == sequenceDirections[seqIdx]) { - // A match; advance both indices to consider next packet in subsequence vs. next packet in sequence. - subseqIdx++; - seqIdx++; - if (subseqIdx == subsequence.size()) { - // We managed to match the entire subsequence in sequence. - // Return the sublist of sequence that matches subsequence. - /* - * TODO: - * ASSUMES THE BACKING LIST (i.e., 'sequence') IS _NOT_ STRUCTURALLY MODIFIED, hence may not work - * for live traces! - */ - return Optional.of(sequence.subList(seqIdx - subsequence.size(), seqIdx)); - } - } else { - // Mismatch. - if (subseqIdx > 0) { - /* - * If we managed to match parts of subsequence, we restart the search for subsequence in sequence at - * the index of sequence where the current mismatch occurred. I.e., we must reset subseqIdx, but - * leave seqIdx untouched. - */ - subseqIdx = 0; - } else { - /* - * First packet of subsequence didn't match packet at seqIdx of sequence, so we move forward in - * sequence, i.e., we continue the search for subsequence in sequence starting at index seqIdx+1 of - * sequence. - */ - seqIdx++; - } - } - } - return Optional.empty(); - } - - /** - * Given a cluster, produces a pruned version of that cluster. In the pruned version, there are no duplicate cluster - * members. Two cluster members are considered identical if their packets lengths and packet directions are - * identical. The resulting pruned cluster is unmodifiable (this applies to both the outermost list as well as the - * nested lists) in order to preserve its integrity when exposed to external code (e.g., through - * {@link #getCluster()}). - * - * @param cluster A cluster to prune. - * @return The resulting pruned cluster. - */ - private final List> pruneCluster(List> cluster) { - List> prunedCluster = new ArrayList<>(); - for (List originalClusterSeq : cluster) { - boolean alreadyPresent = false; - for (List prunedClusterSeq : prunedCluster) { - Optional> duplicate = findSubsequenceInSequence(originalClusterSeq, prunedClusterSeq, - mClusterMemberDirections, mClusterMemberDirections); - if (duplicate.isPresent()) { - alreadyPresent = true; - break; - } - } - if (!alreadyPresent) { - prunedCluster.add(Collections.unmodifiableList(originalClusterSeq)); - } - } - return Collections.unmodifiableList(prunedCluster); - } - - /** - * Given a {@code List}, generate a {@code Conversation.Direction[]} such that each entry in the - * resulting {@code Conversation.Direction[]} specifies the direction of the {@link PcapPacket} at the corresponding - * index in the input list. - * @param packets The list of packets for which to construct a corresponding array of packet directions. - * @param routerWanIp The IP of the router's WAN port. This is used for determining the direction of packets when - * the traffic is captured just outside the local network (at the ISP side of the router). Set to - * {@code null} if {@code packets} stem from traffic captured within the local network. - * @return A {@code Conversation.Direction[]} specifying the direction of the {@link PcapPacket} at the - * corresponding index in {@code packets}. - */ - private static Conversation.Direction[] getPacketDirections(List packets, String routerWanIp) { - Conversation.Direction[] directions = new Conversation.Direction[packets.size()]; - for (int i = 0; i < packets.size(); i++) { - PcapPacket pkt = packets.get(i); - if (getSourceIp(pkt).equals(getDestinationIp(pkt))) { - // Sanity check: we shouldn't be processing loopback traffic - throw new AssertionError("loopback traffic detected"); - } - if (isSrcIpLocal(pkt) || getSourceIp(pkt).equals(routerWanIp)) { - directions[i] = Conversation.Direction.CLIENT_TO_SERVER; - } else if (isDstIpLocal(pkt) || getDestinationIp(pkt).equals(routerWanIp)) { - directions[i] = Conversation.Direction.SERVER_TO_CLIENT; - } else { - //throw new IllegalArgumentException("no local IP or router WAN port IP found, can't detect direction"); - } - } - return directions; - } - - /** - * Interface used by client code to register for receiving a notification whenever the {@link ClusterMatcher} - * detects traffic that is similar to the traffic that makes up the cluster returned by - * {@link ClusterMatcher#getCluster()}. - */ - interface ClusterMatchObserver { - /** - * Callback that is invoked whenever a sequence that is similar to a sequence associated with the cluster (i.e., - * a sequence is a member of the cluster) is detected in the traffic that the associated {@link ClusterMatcher} - * observes. - * @param clusterMatcher The {@link ClusterMatcher} that detected a match (classified traffic as pertaining to - * its associated cluster). - * @param match The traffic that was deemed to match the cluster associated with {@code clusterMatcher}. - */ - void onMatch(ClusterMatcher clusterMatcher, List match); - } - -} diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/SignatureDetector.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/SignatureDetector.java deleted file mode 100644 index 9d8b4d2..0000000 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/SignatureDetector.java +++ /dev/null @@ -1,662 +0,0 @@ -package edu.uci.iotproject.detection; - -import edu.uci.iotproject.analysis.TriggerTrafficExtractor; -import edu.uci.iotproject.analysis.UserAction; -import edu.uci.iotproject.io.PcapHandleReader; -import edu.uci.iotproject.util.PrintUtils; -import org.jgrapht.GraphPath; -import org.jgrapht.alg.shortestpath.DijkstraShortestPath; -import org.jgrapht.graph.DefaultWeightedEdge; -import org.jgrapht.graph.SimpleDirectedWeightedGraph; -import org.pcap4j.core.*; - -import java.time.Duration; -import java.time.Instant; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.time.format.FormatStyle; -import java.util.*; -import java.util.function.Consumer; -import java.util.stream.Collectors; - -/** - * Detects an event signature that spans one or multiple TCP connections. - * - * @author Janus Varmarken {@literal } - * @author Rahmadi Trimananda {@literal } - */ -public class SignatureDetector implements PacketListener, ClusterMatcher.ClusterMatchObserver { - - // Test client - public static void main(String[] args) throws PcapNativeException, NotOpenException { -// if (args.length < 3) { -// String errMsg = String.format("Usage: %s inputPcapFile onSignatureFile offSignatureFile", -// SignatureDetector.class.getSimpleName()); -// System.out.println(errMsg); -// return; -// } -// final String inputPcapFile = args[0]; -// final String onSignatureFile = args[1]; -// final String offSignatureFile = args[2]; - - String path = "/scratch/July-2018"; // Rahmadi -// String path = "/Users/varmarken/temp/UCI IoT Project/experiments"; // Janus -// String path = "/home/jvarmark/iot_project/datasets"; // Hera (server) -// String path = "/raid/varmarken/iot_project/datasets"; // Zeus (server) - - // No activity test - //final String inputPcapFile = path + "/evaluation/no-activity/no-activity.wlan1.pcap"; - - // D-Link Siren experiment -// final String inputPcapFile = path + "/evaluation/dlink-siren/dlink-siren.data.wlan1.pcap"; -// final String inputPcapFile = path + "/evaluation/dlink-siren/dlink-siren.eth0.local.pcap"; - // D-Link Siren DEVICE signatures -// final String onSignatureFile = path + "/2018-08/dlink-siren/onSignature-DLink-Siren-device.sig"; -// final String offSignatureFile = path + "/2018-08/dlink-siren/offSignature-DLink-Siren-device.sig"; - // D-Link Siren PHONE signatures -// final String onSignatureFile = path + "/2018-08/dlink-siren/onSignature-DLink-Siren-phone.sig"; -// final String offSignatureFile = path + "/2018-08/dlink-siren/offSignature-DLink-Siren-phone.sig"; - // TODO: EXPERIMENT - November 19, 2018 - // Hue Bulb experiment -// final String inputPcapFile = path + "/2018-08/hue-bulb/hue-bulb.wlan1.local.pcap"; - // Hue Bulb PHONE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/hue-bulb/signatures/hue-bulb-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/hue-bulb/signatures/hue-bulb-offSignature-phone-side.sig"; - - /* - // Kwikset Doorlock Sep 12 experiment -// final String inputPcapFile = path + "/evaluation/kwikset-doorlock/kwikset-doorlock.data.wlan1.pcap"; - final String inputPcapFile = path + "/evaluation/kwikset-doorlock/kwikset-doorlock.data.eth0.pcap"; -// // Kwikset Doorlock PHONE signatures - final String onSignatureFile = path + "/2018-08/kwikset-doorlock/onSignature-Kwikset-Doorlock-phone-new.sig"; - final String offSignatureFile = path + "/2018-08/kwikset-doorlock/offSignature-Kwikset-Doorlock-phone-new.sig"; - */ - - // D-Link Plug experiment - //final String inputPcapFile = path + "/evaluation/dlink/dlink-plug.data.wlan1.pcap"; -// final String inputPcapFile = path + "/evaluation/dlink/dlink-plug.data.eth0.pcap"; - - // D-Link Plug DEVICE signatures -// final String onSignatureFile = path + "/2018-07/dlink/onSignature-DLink-Plug-device.sig"; -// final String offSignatureFile = path + "/2018-07/dlink/offSignature-DLink-Plug-device.sig"; - // D-Link Plug PHONE signatures -// final String onSignatureFile = path + "/2018-07/dlink/onSignature-DLink-Plug-phone.sig"; -// final String offSignatureFile = path + "/2018-07/dlink/offSignature-DLink-Plug-phone.sig"; - - // TODO: The following are negative tests against the PCAP file from UNSW -// final String inputPcapFile = path + "/UNSW/16-10-04.pcap"; // TODO: Seems to be broken! Zero-payload! -// final String inputPcapFile = path + "/UNSW/16-10-12.pcap"; - -// final String inputPcapFile = path + "/UNSW/16-09-28.pcap"; // TODO: Seems to be broken! Zero-payload! -// final String inputPcapFile = path + "/UNSW/16-10-02.pcap"; // TODO: Seems to be broken! -// final String inputPcapFile = path + "/UNSW/16-10-03.pcap"; // TODO: Seems to be broken! -// final String inputPcapFile = path + "/UNSW/16-10-04-a.pcap"; // TODO: Seems to be broken! Zero-payload! -// final String inputPcapFile = path + "/UNSW/16-10-04-b.pcap"; // TODO: Seems to be broken! Zero-payload! -// final String inputPcapFile = path + "/UNSW/16-10-07.pcap"; // TODO: Seems to be broken! -// final String inputPcapFile = path + "/UNSW/16-10-08.pcap"; // TODO: Seems to be broken! -// final String inputPcapFile = path + "/UNSW/16-10-09.pcap"; // TODO: Seems to be broken! -// final String inputPcapFile = path + "/UNSW/16-10-10.pcap"; // TODO: Seems to be broken! -// final String inputPcapFile = path + "/UNSW/16-10-11.pcap"; // TODO: Seems to be broken! - // TODO: The following one is very long!!! - Split into smaller files! -// final String inputPcapFile = path + "/UNSW/16-10-12-a.pcap"; -// final String inputPcapFile = path + "/UNSW/16-10-12-b.pcap"; -// final String inputPcapFile = path + "/UNSW/16-10-12-c.pcap"; -// final String inputPcapFile = path + "/UNSW/16-10-12-d.pcap"; - -// final String inputPcapFile = path + "/UNSW/16-09-23.pcap"; -// final String inputPcapFile = path + "/UNSW/16-09-24.pcap"; -// final String inputPcapFile = path + "/UNSW/16-09-25.pcap"; -// final String inputPcapFile = path + "/UNSW/16-09-26.pcap"; -// final String inputPcapFile = path + "/UNSW/16-09-27.pcap"; -// final String inputPcapFile = path + "/UNSW/16-09-29.pcap"; -// final String inputPcapFile = path + "/UNSW/16-10-01.pcap"; -// final String inputPcapFile = path + "/UNSW/16-10-06.pcap"; - // Negative test: dataset from UNB -// final String inputPcapFile = path + "/evaluation/negative-datasets/UNB/Monday-WorkingHours_one-local-endpoint.pcap"; - - // TODO: The following are tests for signatures against training data - - // D-Link Plug experiment -// final String inputPcapFile = path + "/training/dlink-plug/wlan1/dlink-plug.wlan1.local.pcap"; - // D-Link Plug DEVICE signatures -// final String onSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-onSignature-device-side.sig"; -// final String offSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-offSignature-device-side.sig"; - // D-Link Plug PHONE signatures -// final String onSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-offSignature-phone-side.sig"; - - // TODO: EXPERIMENT - November 7, 2018 - // D-Link Plug experiment - //final String inputPcapFile = path + "/experimental_result/standalone/dlink-plug/wlan1/dlink-plug.wlan1.local.pcap"; - //final String inputPcapFile = path + "/experimental_result/smarthome/dlink-plug/wlan1/dlink-plug.wlan1.detection.pcap"; - //final String inputPcapFile = path + "/experimental_result/smarthome/dlink-plug/eth0/dlink-plug.eth0.detection.pcap"; - // D-Link Plug DEVICE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-onSignature-device-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-offSignature-device-side.sig"; - // D-Link Plug PHONE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-offSignature-phone-side.sig"; - - // TODO: EXPERIMENT - November 9, 2018 - // D-Link Siren experiment - //final String inputPcapFile = path + "/experimental_result/standalone/dlink-siren/wlan1/dlink-siren.wlan1.local.pcap"; - //final String inputPcapFile = path + "/experimental_result/smarthome/dlink-siren/wlan1/dlink-siren.wlan1.detection.pcap"; -// final String inputPcapFile = path + "/experimental_result/smarthome/dlink-siren/eth0/dlink-siren.eth0.detection.pcap"; - // D-Link Siren DEVICE signatures - // TODO: The device signature does not have pairs---only one packet which is 216, so we don't consider this as a signature -// final String onSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-onSignature-device-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-offSignature-device-side.sig"; - // D-Link Siren PHONE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-offSignature-phone-side.sig"; -// final String onSignatureFile = path + "/training/signatures/dlink-siren/dlink-siren-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/training/signatures/dlink-siren/dlink-siren-offSignature-phone-side.sig"; - - // TP-Link Plug experiment -//// final String inputPcapFile = path + "/training/tplink-plug/wlan1/tplink-plug.wlan1.local.pcap"; -//// final String inputPcapFile = path + "/experimental_result/wifi-Sniffer/tests2/airtool_2019-01-04_11.08.45.AM.pcap"; -// final String inputPcapFile = path + "/experimental_result/wifi-Sniffer/tests2/command-frames-only.pcap"; -// // TP-Link Plug DEVICE signatures -// final String onSignatureFile = path + "/training/tplink-plug/signatures/tplink-plug-onSignature-device-side.sig"; -// final String offSignatureFile = path + "/training/tplink-plug/signatures/tplink-plug-offSignature-device-side.sig"; - // TODO: EXPERIMENT - November 8, 2018 - // TP-Link Plug experiment -// final String inputPcapFile = path + "/experimental_result/standalone/tplink-plug/wlan1/tplink-plug.wlan1.local.pcap"; -// final String inputPcapFile = path + "/experimental_result/standalone/tplink-plug/eth0/tplink-plug.eth0.local.pcap"; -// final String inputPcapFile = path + "/experimental_result/smarthome/tplink-plug/wlan1/tplink-plug.wlan1.detection.pcap"; - //final String inputPcapFile = path + "/experimental_result/smarthome/tplink-plug/eth0/tplink-plug.eth0.detection.pcap"; - // TP-Link Plug DEVICE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-device-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-device-side.sig"; -// final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-device-side-outbound.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-device-side-outbound.sig"; - // TP-Link Plug PHONE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-phone-side.sig"; - - // Arlo camera experiment -// final String inputPcapFile = path + "/training/arlo-camera/wlan1/arlo-camera.wlan1.local.pcap"; -//// // TP-Link Plug DEVICE signatures -// final String onSignatureFile = path + "/training/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/training/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig"; - // TODO: EXPERIMENT - November 13, 2018 - // Arlo Camera experiment -// final String inputPcapFile = path + "/experimental_result/standalone/arlo-camera/wlan1/arlo-camera.wlan1.local.pcap"; -// final String inputPcapFile = path + "/experimental_result/standalone/arlo-camera/eth0/arlo-camera.eth0.local.pcap"; -// final String inputPcapFile = path + "/experimental_result/smarthome/arlo-camera/wlan1/arlo-camera.wlan1.detection.pcap"; -// final String inputPcapFile = path + "/experimental_result/smarthome/arlo-camera/eth0/arlo-camera.eth0.detection.pcap"; -// final String inputPcapFile = path + "/training/arlo-camera/eth0/arlo-camera.eth0.local.pcap"; - // Arlo Camera PHONE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig"; - - // Amazon Alexa experiment -// final String inputPcapFile = path + "/training/amazon-alexa/wlan1/alexa2.wlan1.local.pcap"; -// // TP-Link Plug DEVICE signatures -// final String onSignatureFile = path + "/training/amazon-alexa/signatures/amazon-alexa-onSignature-device-side.sig"; -// final String offSignatureFile = path + "/training/amazon-alexa/signatures/amazon-alexa-offSignature-device-side.sig"; - - // SmartThings Plug experiment -// final String inputPcapFile = path + "/training/st-plug/wlan1/st-plug.wlan1.local.pcap"; -// // SmartThings Plug DEVICE signatures -// //final String onSignatureFile = path + "/training/st-plug/signatures/st-plug-onSignature-device-side.sig"; -// //final String offSignatureFile = path + "/training/st-plug/signatures/st-plug-offSignature-device-side.sig"; -// // SmartThings Plug PHONE signatures -// final String onSignatureFile = path + "/training/st-plug/signatures/st-plug-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/training/st-plug/signatures/st-plug-offSignature-phone-side.sig"; - // TODO: EXPERIMENT - November 12, 2018 - // SmartThings Plug experiment -// final String inputPcapFile = path + "/experimental_result/standalone/st-plug/wlan1/st-plug.wlan1.local.pcap"; -// final String inputPcapFile = path + "/experimental_result/standalone/st-plug/eth0/st-plug.eth0.local.pcap"; -// //final String inputPcapFile = path + "/experimental_result/smarthome/st-plug/wlan1/st-plug.wlan1.detection.pcap"; -// final String inputPcapFile = path + "/experimental_result/smarthome/st-plug/eth0/st-plug.eth0.detection.pcap"; -// // SmartThings Plug PHONE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/st-plug/signatures/st-plug-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/st-plug/signatures/st-plug-offSignature-phone-side.sig"; -// final String onSignatureFile = path + "/training/signatures/st-plug/st-plug-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/training/signatures/st-plug/st-plug-offSignature-phone-side.sig"; - - // TODO: EXPERIMENT - January 9, 2018 - // Blossom Sprinkler experiment -// final String inputPcapFile = path + "/experimental_result/standalone/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.local.pcap"; - final String inputPcapFile = path + "/experimental_result/smarthome/blossom-sprinkler/eth0/blossom-sprinkler.eth0.detection.pcap"; -// final String inputPcapFile = path + "/experimental_result/smarthome/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.detection.pcap"; - // Blossom Sprinkler DEVICE signatures - final String onSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig"; - final String offSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig"; - // Blossom Sprinkler PHONE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-phone-side.sig"; - - // LiFX Bulb experiment -// final String inputPcapFile = path + "/training/lifx-bulb/wlan1/lifx-bulb.wlan1.local.pcap"; -// // LiFX Bulb DEVICE signatures -// final String onSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-onSignature-device-side.sig"; -// final String offSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-offSignature-device-side.sig"; - // LiFX Bulb PHONE signatures -// final String onSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-offSignature-phone-side.sig"; - - // Blossom Sprinkler experiment -// //final String inputPcapFile = path + "/training/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.local.pcap"; -// final String inputPcapFile = path + "/2018-08/blossom/blossom.wlan1.local.pcap"; -// //final String inputPcapFile = path + "/training/blossom-sprinkler/eth0/blossom-sprinkler.eth0.local.pcap"; -// // Blossom Sprinkler DEVICE signatures -// final String onSignatureFile = path + "/training/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig"; -// final String offSignatureFile = path + "/training/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig"; - - // Nest Thermostat experiment -// final String inputPcapFile = path + "/training/nest-thermostat/wlan1/nest-thermostat.wlan1.local.pcap"; -// // Nest Thermostat DEVICE signatures -//// final String onSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-onSignature-device-side.sig"; -//// final String offSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-offSignature-device-side.sig"; -// // Nest Thermostat PHONE signatures -// final String onSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-offSignature-phone-side.sig"; - // TODO: EXPERIMENT - November 15, 2018 - // Nest Thermostat experiment -// final String inputPcapFile = path + "/experimental_result/standalone/nest-thermostat/wlan1/nest-thermostat.wlan1.local.pcap"; -//// final String inputPcapFile = path + "/experimental_result/standalone/nest-thermostat/eth0/nest-thermostat.eth0.local.pcap"; -//// final String inputPcapFile = path + "/experimental_result/smarthome/nest-thermostat/wlan1/nest-thermostat.wlan1.detection.pcap"; -// final String inputPcapFile = path + "/experimental_result/smarthome/nest-thermostat/eth0/nest-thermostat.eth0.detection.pcap"; -//// // Nest Thermostat PHONE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/nest-thermostat/signatures/nest-thermostat-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/nest-thermostat/signatures/nest-thermostat-offSignature-phone-side.sig"; - - /* - // Hue Bulb experiment - final String inputPcapFile = path + "/training/hue-bulb/wlan1/hue-bulb.wlan1.local.pcap"; - // Hue Bulb PHONE signatures - final String onSignatureFile = path + "/training/hue-bulb/signatures/hue-bulb-onSignature-phone-side.sig"; - final String offSignatureFile = path + "/training/hue-bulb/signatures/hue-bulb-offSignature-phone-side.sig"; - */ - - - - // TP-Link Bulb experiment -// final String inputPcapFile = path + "/training/tplink-bulb/wlan1/tplink-bulb.wlan1.local.pcap"; -// // TP-Link Bulb PHONE signatures -// final String onSignatureFile = path + "/training/tplink-bulb/signatures/tplink-bulb-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/training/tplink-bulb/signatures/tplink-bulb-offSignature-phone-side.sig"; - // TODO: EXPERIMENT - November 16, 2018 - // TP-Link Bulb experiment -// final String inputPcapFile = path + "/experimental_result/standalone/tplink-bulb/wlan1/tplink-bulb.wlan1.local.pcap"; -// final String inputPcapFile = path + "/experimental_result/standalone/tplink-bulb/eth0/tplink-bulb.eth0.local.pcap"; -// final String inputPcapFile = path + "/experimental_result/smarthome/tplink-bulb/wlan1/tplink-bulb.wlan1.detection.pcap"; -//// final String inputPcapFile = path + "/experimental_result/smarthome/tplink-bulb/eth0/tplink-bulb.eth0.detection.pcap"; -// // TP-Link Bulb PHONE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/tplink-bulb/signatures/tplink-bulb-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/tplink-bulb/signatures/tplink-bulb-offSignature-phone-side.sig"; - - /* - // WeMo Plug experiment - final String inputPcapFile = path + "/training/wemo-plug/wlan1/wemo-plug.wlan1.local.pcap"; - // WeMo Plug PHONE signatures - final String onSignatureFile = path + "/training/wemo-plug/signatures/wemo-plug-onSignature-device-side.sig"; - final String offSignatureFile = path + "/training/wemo-plug/signatures/wemo-plug-offSignature-device-side.sig"; - */ - // TODO: EXPERIMENT - November 20, 2018 - // WeMo Plug experiment -// final String inputPcapFile = path + "/experimental_result/standalone/wemo-plug/wlan1/wemo-plug.wlan1.local.pcap"; -// final String inputPcapFile = path + "/experimental_result/standalone/wemo-plug/eth0/wemo-plug.eth0.local.pcap"; - // TODO: WE HAVE 4 ADDITIONAL EVENTS (TRIGGERED MANUALLY), SO WE JUST IGNORE THEM BECAUSE THEY HAPPENED BEFORE - // TODO: THE ACTUAL TRIGGERS -// final String inputPcapFile = path + "/experimental_result/smarthome/wemo-plug/wlan1/wemo-plug.wlan1.detection.pcap"; -//// final String inputPcapFile = path + "/experimental_result/smarthome/wemo-plug/eth0/wemo-plug.eth0.detection.pcap"; -// // WeMo Plug PHONE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/wemo-plug/signatures/wemo-plug-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/wemo-plug/signatures/wemo-plug-offSignature-phone-side.sig"; - - /* - // WeMo Insight Plug experiment - final String inputPcapFile = path + "/training/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.local.pcap"; - // WeMo Insight Plug PHONE signatures - final String onSignatureFile = path + "/training/wemo-insight-plug/signatures/wemo-insight-plug-onSignature-device-side.sig"; - final String offSignatureFile = path + "/training/wemo-insight-plug/signatures/wemo-insight-plug-offSignature-device-side.sig"; - */ - // TODO: EXPERIMENT - November 21, 2018 - // WeMo Insight Plug experiment -// final String inputPcapFile = path + "/experimental_result/standalone/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.local.pcap"; -// final String inputPcapFile = path + "/experimental_result/standalone/wemo-insight-plug/eth0/wemo-insight-plug.eth0.local.pcap"; - // TODO: WE HAVE 1 ADDITIONAL EVENT (FROM WEMO PLUG) -// final String inputPcapFile = path + "/experimental_result/smarthome/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.detection.pcap"; -// final String inputPcapFile = path + "/experimental_result/smarthome/wemo-insight-plug/eth0/wemo-insight-plug.eth0.detection.pcap"; - // WeMo Insight Plug PHONE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/wemo-insight-plug/signatures/wemo-insight-plug-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/wemo-insight-plug/signatures/wemo-insight-plug-offSignature-phone-side.sig"; - - - // Kwikset Doorlock Sep 12 experiment -// final String inputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset3.wlan1.local.pcap"; -// // Kwikset Doorlock PHONE signatures -// final String onSignatureFile = path + "/2018-08/kwikset-doorlock/onSignature-Kwikset-Doorlock-phone.sig"; -// final String offSignatureFile = path + "/2018-08/kwikset-doorlock/offSignature-Kwikset-Doorlock-phone.sig"; - // TODO: EXPERIMENT - November 10, 2018 - // Kwikset Door lock experiment -// final String inputPcapFile = path + "/experimental_result/standalone/kwikset-doorlock/wlan1/kwikset-doorlock.wlan1.local.pcap"; -// //final String inputPcapFile = path + "/experimental_result/smarthome/kwikset-doorlock/wlan1/kwikset-doorlock.wlan1.detection.pcap"; -// final String inputPcapFile = path + "/experimental_result/smarthome/kwikset-doorlock/eth0/kwikset-doorlock.eth0.detection.pcap"; -//// // Kwikset Door lock PHONE signatures -// final String onSignatureFile = path + "/experimental_result/standalone/kwikset-doorlock/signatures/kwikset-doorlock-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/experimental_result/standalone/kwikset-doorlock/signatures/kwikset-doorlock-offSignature-phone-side.sig"; -// final String onSignatureFile = path + "/training/signatures/kwikset-doorlock/kwikset-doorlock-onSignature-phone-side.sig"; -// final String offSignatureFile = path + "/training/signatures/kwikset-doorlock/kwikset-doorlock-offSignature-phone-side.sig"; - - - - // D-Link Siren experiment -// final String inputPcapFile = path + "/2018-08/dlink-siren/dlink-siren.wlan1.local.pcap"; - // D-Link Siren DEVICE signatures - //final String onSignatureFile = path + "/2018-08/dlink-siren/onSignature-DLink-Siren-device.sig"; - //final String offSignatureFile = path + "/2018-08/dlink-siren/offSignature-DLink-Siren-device.sig"; - // D-Link Siren PHONE signatures -// final String onSignatureFile = path + "/2018-08/dlink-siren/onSignature-DLink-Siren-phone.sig"; -// final String offSignatureFile = path + "/2018-08/dlink-siren/offSignature-DLink-Siren-phone.sig"; - - - // Output file names used (to make it easy to catch if one forgets to change them) - System.out.println("ON signature file in use is " + onSignatureFile); - System.out.println("OFF signature file in use is " + offSignatureFile); - System.out.println("PCAP file that is the target of detection is " + inputPcapFile); - - List>> onSignature = PrintUtils.deserializeSignatureFromFile(onSignatureFile); - List>> offSignature = PrintUtils.deserializeSignatureFromFile(offSignatureFile); - - // LAN -// SignatureDetector onDetector = new SignatureDetector(onSignature, null); -// SignatureDetector offDetector = new SignatureDetector(offSignature, null); - // WAN - SignatureDetector onDetector = new SignatureDetector(onSignature, "128.195.205.105"); - SignatureDetector offDetector = new SignatureDetector(offSignature, "128.195.205.105"); - - final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofLocalizedDateTime(FormatStyle.MEDIUM). - withLocale(Locale.US).withZone(ZoneId.of("America/Los_Angeles")); - - // Outputs information about a detected event to std.out - final Consumer outputter = ua -> { - String eventDescription; - switch (ua.getType()) { - case TOGGLE_ON: - eventDescription = "ON"; - break; - case TOGGLE_OFF: - eventDescription = "OFF"; - break; - default: - throw new AssertionError("unhandled event type"); - } - //String output = String.format("[ !!! %s SIGNATURE DETECTED at %s !!! ]", - // eventDescription, dateTimeFormatter.format(ua.getTimestamp())); - String output = String.format("%s", - dateTimeFormatter.format(ua.getTimestamp())); - System.out.println(output); - }; - - // Let's create observers that construct a UserAction representing the detected event. - final List detectedEvents = new ArrayList<>(); - onDetector.addObserver((searched, match) -> { - PcapPacket firstPkt = match.get(0).get(0); - detectedEvents.add(new UserAction(UserAction.Type.TOGGLE_ON, firstPkt.getTimestamp())); - }); - offDetector.addObserver((searched, match) -> { - PcapPacket firstPkt = match.get(0).get(0); - detectedEvents.add(new UserAction(UserAction.Type.TOGGLE_OFF, firstPkt.getTimestamp())); - }); - - PcapHandle handle; - try { - handle = Pcaps.openOffline(inputPcapFile, PcapHandle.TimestampPrecision.NANO); - } catch (PcapNativeException pne) { - handle = Pcaps.openOffline(inputPcapFile); - } - PcapHandleReader reader = new PcapHandleReader(handle, p -> true, onDetector, offDetector); - reader.readFromHandle(); - - // TODO: need a better way of triggering detection than this... - onDetector.mClusterMatchers.forEach(cm -> cm.performDetection()); - offDetector.mClusterMatchers.forEach(cm -> cm.performDetection()); - - // Sort the list of detected events by timestamp to make it easier to compare it line-by-line with the trigger - // times file. - Collections.sort(detectedEvents, Comparator.comparing(UserAction::getTimestamp)); - - // Output the detected events - detectedEvents.forEach(outputter); - - System.out.println("Number of detected events of type " + UserAction.Type.TOGGLE_ON + ": " + - detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_ON).count()); - System.out.println("Number of detected events of type " + UserAction.Type.TOGGLE_OFF + ": " + - detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_OFF).count()); - - - // TODO: Temporary clean up until we clean the pipeline -// List cleanedDetectedEvents = SignatureDetector.removeDuplicates(detectedEvents); -// cleanedDetectedEvents.forEach(outputter); - } - - /** - * The signature that this {@link SignatureDetector} is searching for. - */ - private final List>> mSignature; - - /** - * The {@link ClusterMatcher}s in charge of detecting each individual sequence of packets that together make up the - * the signature. - */ - private final List mClusterMatchers; - - /** - * For each {@code i} ({@code i >= 0 && i < pendingMatches.length}), {@code pendingMatches[i]} holds the matches - * found by the {@link ClusterMatcher} at {@code mClusterMatchers.get(i)} that have yet to be "consumed", i.e., - * have yet to be included in a signature detected by this {@link SignatureDetector} (a signature can be encompassed - * of multiple packet sequences occurring shortly after one another on multiple connections). - */ - private final List>[] pendingMatches; - - /** - * Maps a {@link ClusterMatcher} to its corresponding index in {@link #pendingMatches}. - */ - private final Map mClusterMatcherIds; - - private final List mObservers = new ArrayList<>(); - - /** - * Remove duplicates in {@code List} of {@code UserAction} objects. We need to clean this up for user actions - * that appear multiple times. - * TODO: This static method is probably just for temporary and we could get rid of this after we clean up - * TODO: the pipeline - * - * @param listUserAction A {@link List} of {@code UserAction}. - * - */ - public static List removeDuplicates(List listUserAction) { - - // Iterate and check for duplicates (check timestamps) - Set epochSecondSet = new HashSet<>(); - // Create a target list for cleaned up list - List listUserActionClean = new ArrayList<>(); - for(UserAction userAction : listUserAction) { - // Don't insert if any duplicate is found - if(!epochSecondSet.contains(userAction.getTimestamp().getEpochSecond())) { - listUserActionClean.add(userAction); - epochSecondSet.add(userAction.getTimestamp().getEpochSecond()); - } - } - return listUserActionClean; - } - - public SignatureDetector(List>> searchedSignature, String routerWanIp) { - // note: doesn't protect inner lists from changes :'( - mSignature = Collections.unmodifiableList(searchedSignature); - // Generate corresponding/appropriate ClusterMatchers based on the provided signature - List clusterMatchers = new ArrayList<>(); - for (List> cluster : mSignature) { - clusterMatchers.add(new ClusterMatcher(cluster, routerWanIp, this)); - } - mClusterMatchers = Collections.unmodifiableList(clusterMatchers); - - // < exploratory > - pendingMatches = new List[mClusterMatchers.size()]; - for (int i = 0; i < pendingMatches.length; i++) { - pendingMatches[i] = new ArrayList<>(); - } - Map clusterMatcherIds = new HashMap<>(); - for (int i = 0; i < mClusterMatchers.size(); i++) { - clusterMatcherIds.put(mClusterMatchers.get(i), i); - } - mClusterMatcherIds = Collections.unmodifiableMap(clusterMatcherIds); - } - - public void addObserver(SignatureDetectionObserver observer) { - mObservers.add(observer); - } - - public boolean removeObserver(SignatureDetectionObserver observer) { - return mObservers.remove(observer); - } - - @Override - public void gotPacket(PcapPacket packet) { - // simply delegate packet reception to all ClusterMatchers. - mClusterMatchers.forEach(cm -> cm.gotPacket(packet)); - } - - @Override - public void onMatch(ClusterMatcher clusterMatcher, List match) { - // Add the match at the corresponding index - pendingMatches[mClusterMatcherIds.get(clusterMatcher)].add(match); - checkSignatureMatch(); - } - - private void checkSignatureMatch() { - // << Graph-based approach using Balint's idea. >> - // This implementation assumes that the packets in the inner lists (the sequences) are ordered by asc timestamp. - - // There cannot be a signature match until each ClusterMatcher has found a match of its respective sequence. - if (Arrays.stream(pendingMatches).noneMatch(l -> l.isEmpty())) { - // Construct the DAG - final SimpleDirectedWeightedGraph graph = - new SimpleDirectedWeightedGraph<>(DefaultWeightedEdge.class); - // Add a vertex for each match found by all ClusterMatchers - // And maintain an array to keep track of what cluster matcher each vertex corresponds to - final List[] vertices = new List[pendingMatches.length]; - for (int i = 0; i < pendingMatches.length; i++) { - vertices[i] = new ArrayList<>(); - for (List sequence : pendingMatches[i]) { - Vertex v = new Vertex(sequence); - vertices[i].add(v); // retain reference for later when we are to add edges - graph.addVertex(v); // add to vertex to graph - } - } - // Add dummy source and sink vertices to facilitate search. - final Vertex source = new Vertex(null); - final Vertex sink = new Vertex(null); - graph.addVertex(source); - graph.addVertex(sink); - // The source is connected to all vertices that wrap the sequences detected by ClusterMatcher at index 0. - // Note: zero cost edges as this is just a dummy link to facilitate search from a common start node. - for (Vertex v : vertices[0]) { - DefaultWeightedEdge edge = graph.addEdge(source, v); - graph.setEdgeWeight(edge, 0.0); - } - // Similarly, all vertices that wrap the sequences detected by the last ClusterMatcher of the signature - // are connected to the sink node. - for (Vertex v : vertices[vertices.length-1]) { - DefaultWeightedEdge edge = graph.addEdge(v, sink); - graph.setEdgeWeight(edge, 0.0); - } - // Now link sequences detected by ClusterMatcher at index i to sequences detected by ClusterMatcher at index - // i+1 if they obey the timestamp constraint (i.e., that the latter is later in time than the former). - for (int i = 0; i < vertices.length; i++) { - int j = i + 1; - if (j < vertices.length) { - for (Vertex iv : vertices[i]) { - PcapPacket ivLast = iv.sequence.get(iv.sequence.size()-1); - for (Vertex jv : vertices[j]) { - PcapPacket jvFirst = jv.sequence.get(jv.sequence.size()-1); - if (ivLast.getTimestamp().isBefore(jvFirst.getTimestamp())) { - DefaultWeightedEdge edge = graph.addEdge(iv, jv); - // The weight is the duration of the i'th sequence plus the duration between the i'th - // and i+1'th sequence. - Duration d = Duration. - between(iv.sequence.get(0).getTimestamp(), jvFirst.getTimestamp()); - // Unfortunately weights are double values, so must convert from long to double. - // TODO: need nano second precision? If so, use d.toNanos(). - // TODO: risk of overflow when converting from long to double..? - graph.setEdgeWeight(edge, Long.valueOf(d.toMillis()).doubleValue()); - } - // Alternative version if we cannot assume that sequences are ordered by timestamp: -// if (iv.sequence.stream().max(Comparator.comparing(PcapPacket::getTimestamp)).get() -// .getTimestamp().isBefore(jv.sequence.stream().min( -// Comparator.comparing(PcapPacket::getTimestamp)).get().getTimestamp())) { -// -// } - } - } - } - } - // Graph construction complete, run shortest-path to find a (potential) signature match. - DijkstraShortestPath dijkstra = new DijkstraShortestPath<>(graph); - GraphPath shortestPath = dijkstra.getPath(source, sink); - if (shortestPath != null) { - // The total weight is the duration between the first packet of the first sequence and the last packet - // of the last sequence, so we simply have to compare the weight against the timeframe that we allow - // the signature to span. For now we just use the inclusion window we defined for training purposes. - // Note however, that we must convert back from double to long as the weight is stored as a double in - // JGraphT's API. - if (((long)shortestPath.getWeight()) < TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS) { - // There's a signature match! - // Extract the match from the vertices - List> signatureMatch = new ArrayList<>(); - for(Vertex v : shortestPath.getVertexList()) { - if (v == source || v == sink) { - // Skip the dummy source and sink nodes. - continue; - } - signatureMatch.add(v.sequence); - // As there is a one-to-one correspondence between vertices[] and pendingMatches[], we know that - // the sequence we've "consumed" for index i of the matched signature is also at index i in - // pendingMatches. We must remove it from pendingMatches so that we don't use it to construct - // another signature match in a later call. - pendingMatches[signatureMatch.size()-1].remove(v.sequence); - } - // Declare success: notify observers - mObservers.forEach(obs -> obs.onSignatureDetected(mSignature, - Collections.unmodifiableList(signatureMatch))); - } - } - } - } - - /** - * Used for registering for notifications of signatures detected by a {@link SignatureDetector}. - */ - interface SignatureDetectionObserver { - - /** - * Invoked when the {@link SignatureDetector} detects the presence of a signature in the traffic that it's - * examining. - * @param searchedSignature The signature that the {@link SignatureDetector} reporting the match is searching - * for. - * @param matchingTraffic The actual traffic trace that matches the searched signature. - */ - void onSignatureDetected(List>> searchedSignature, - List> matchingTraffic); - } - - /** - * Encapsulates a {@code List} so as to allow the list to be used as a vertex in a graph while avoiding - * the expensive {@link AbstractList#equals(Object)} calls when adding vertices to the graph. - * Using this wrapper makes the incurred {@code equals(Object)} calls delegate to {@link Object#equals(Object)} - * instead of {@link AbstractList#equals(Object)}. The net effect is a faster implementation, but the graph will not - * recognize two lists that contain the same items--from a value and not reference point of view--as the same - * vertex. However, this is fine for our purposes -- in fact restricting it to reference equality seems more - * appropriate. - */ - private static class Vertex { - private final List sequence; - private Vertex(List wrappedSequence) { - sequence = wrappedSequence; - } - } -} diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer2/Layer2ClusterMatcher.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer2/Layer2ClusterMatcher.java new file mode 100644 index 0000000..b3a88b0 --- /dev/null +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer2/Layer2ClusterMatcher.java @@ -0,0 +1,150 @@ +package edu.uci.iotproject.detection.layer2; + +import edu.uci.iotproject.trafficreassembly.layer2.Layer2FlowReassembler; +import edu.uci.iotproject.trafficreassembly.layer2.Layer2Flow; +import edu.uci.iotproject.trafficreassembly.layer2.Layer2FlowReassemblerObserver; +import edu.uci.iotproject.detection.AbstractClusterMatcher; +import edu.uci.iotproject.trafficreassembly.layer2.Layer2FlowObserver; +import edu.uci.iotproject.io.PcapHandleReader; +import edu.uci.iotproject.util.PrintUtils; +import org.pcap4j.core.*; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * TODO add class documentation. + * + * @author Janus Varmarken + */ +public class Layer2ClusterMatcher extends AbstractClusterMatcher implements Layer2FlowReassemblerObserver, Layer2FlowObserver { + + public static void main(String[] args) throws PcapNativeException, NotOpenException { + final String onSignatureFile = "/Users/varmarken/temp/UCI IoT Project/experiments/training/signatures/tplink-plug/tplink-plug-onSignature-device-side.sig"; + List>> onSignature = PrintUtils.deserializeSignatureFromFile(onSignatureFile); + + + Layer2FlowReassembler flowReassembler = new Layer2FlowReassembler(); + + Layer2ClusterMatcher l2ClusterMatcher = new Layer2ClusterMatcher(onSignature.get(0)); + flowReassembler.addObserver(l2ClusterMatcher); + + final String inputPcapFile = "/Users/varmarken/temp/UCI IoT Project/experiments/2018-07/tplink/tplink.wlan1.local.pcap"; + + PcapHandle handle; + try { + handle = Pcaps.openOffline(inputPcapFile, PcapHandle.TimestampPrecision.NANO); + } catch (PcapNativeException pne) { + handle = Pcaps.openOffline(inputPcapFile); + } + PcapHandleReader reader = new PcapHandleReader(handle, p -> true, flowReassembler); + reader.readFromHandle(); + + + } + + + private final List mSeqMatchers; + + public Layer2ClusterMatcher(List> cluster) { + super(cluster); + // Setup a sequence matcher for each sequence of the pruned cluster + mSeqMatchers = new ArrayList<>(); + mCluster.forEach(seq -> mSeqMatchers.add(new Layer2SequenceMatcher(seq))); + +// for (int i = 0; i < mCluster.size(); i++) { +// +// +// mSeqMatchers[i] = new Layer2SequenceMatcher(mCluster.get(i)); +// +// +// } + } + +// @Override +// public void gotPacket(PcapPacket packet) { +// // Forward the packet to all sequence matchers. +// for (Layer2SequenceMatcher matcher : mSeqMatchers) { +// matcher.gotPacket(packet); +// } +// +// +// } + + + private final Map> mPerFlowSeqMatchers = new HashMap<>(); + + @Override + public void onNewPacket(Layer2Flow flow, PcapPacket newPacket) { + if (mPerFlowSeqMatchers.get(flow) == null) { + // If this is the first time we encounter this flow, we need to set up sequence matchers for it. + List matchers = new ArrayList<>(); + mCluster.forEach(seq -> matchers.add(new Layer2SequenceMatcher(seq))); + mPerFlowSeqMatchers.put(flow, matchers); + } + // Buffer for new sequence matchers that will take over the job of observing for the first packet when a + // sequence matcher advances beyond the first packet. + List newSeqMatchers = new ArrayList<>(); + // Buffer for sequence matchers that have terminated and are to be removed from mPerFlowSeqMatchers. + List terminatedSeqMatchers = new ArrayList<>(); + // Present the new packet to all sequence matchers + for (Layer2SequenceMatcher sm : mPerFlowSeqMatchers.get(flow)) { + boolean matched = sm.matchPacket(newPacket); + if (matched && sm.getMatchedPacketsCount() == 1) { + // Setup a new sequence matcher that matches from the beginning of the sequence so as to keep + // progressing in the sequence matcher that just matched the current packet, while still allowing + // for matches of the full sequence in later traffic. This is to accommodate the case where the + // first packet of a sequence is detected in an early packet, but where the remaining packets of + // that sequence do not appear until way later in time (e.g., if the first packet of the sequence + // by chance is generated from traffic unrelated to the trigger traffic). + // Note that we must store the new sequence matcher in a buffer and add it outside the loop in order to + // prevent concurrent modification exceptions. + newSeqMatchers.add(new Layer2SequenceMatcher(sm.getTargetSequence())); + } + if (matched && sm.getMatchedPacketsCount() == sm.getTargetSequencePacketCount()) { + // This sequence matcher has a match of the sequence it was searching for + // TODO report it.... for now just do a dummy printout. + System.out.println("SEQUENCE MATCHER HAS A MATCH AT " + sm.getMatchedPackets().get(0).getTimestamp()); + // Mark the sequence matcher for removal. No need to create a replacement one as we do that whenever the + // first packet of the sequence is matched (see above). + terminatedSeqMatchers.add(sm); + } + } + // Add the new sequence matchers, if any. + mPerFlowSeqMatchers.get(flow).addAll(newSeqMatchers); + // Remove the terminated sequence matchers, if any. + mPerFlowSeqMatchers.get(flow).removeAll(terminatedSeqMatchers); + } + + + @Override + protected List> pruneCluster(List> cluster) { + // Note: we assume that all sequences in the input cluster are of the same length and that their packet + // directions are identical. + List> prunedCluster = new ArrayList<>(); + for (List originalClusterSeq : cluster) { + boolean alreadyPresent = prunedCluster.stream().anyMatch(pcPkts -> { + for (int i = 0; i < pcPkts.size(); i++) { + if (pcPkts.get(i).getOriginalLength() != originalClusterSeq.get(i).getOriginalLength()) { + return false; + } + } + return true; + }); + if (!alreadyPresent) { + // Add the sequence if not already present in the pruned cluster. + prunedCluster.add(originalClusterSeq); + } + } + return prunedCluster; + } + + + @Override + public void onNewFlow(Layer2FlowReassembler reassembler, Layer2Flow newFlow) { + // Subscribe to the new flow to get updates whenever a new packet pertaining to the flow is processed. + newFlow.addFlowObserver(this); + } +} diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SequenceMatcher.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SequenceMatcher.java new file mode 100644 index 0000000..10ae34e --- /dev/null +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer2/Layer2SequenceMatcher.java @@ -0,0 +1,165 @@ +package edu.uci.iotproject.detection.layer2; + +import edu.uci.iotproject.analysis.TriggerTrafficExtractor; +import edu.uci.iotproject.util.PcapPacketUtils; +import org.pcap4j.core.PcapPacket; +import org.pcap4j.util.MacAddress; + +import java.util.ArrayList; +import java.util.List; + +/** + * Attempts to detect the presence of a specific packet sequence in the set of packets provided through multiple calls + * to {@link #matchPacket(PcapPacket)}, considering only layer 2 information. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class Layer2SequenceMatcher { + + /** + * The sequence this {@link Layer2SequenceMatcher} is searching for. + */ + private final List mSequence; + + /** + * Buffer of actual packets seen so far that match the searched sequence (i.e., constitutes a subsequence of the + * searched sequence). + */ + private final List mMatchedPackets = new ArrayList<>(); + + /** + * Models the directions of packets in {@link #mSequence}. As the sequence matcher assumes that it is only presented + * with packet from a single flow (packets exchanged between two devices), we can model the packet directions with a + * single bit. We don't have any notion "phone to device" or "device to phone" as we don't know the MAC addresses + * of devices in advance during matching. + */ + private final boolean[] mPacketDirections; + + /** + * Create a {@code Layer2SequenceMatcher}. + * @param sequence The sequence to match against (search for). + */ + public Layer2SequenceMatcher(List sequence) { + mSequence = sequence; + // Compute packet directions for sequence. + mPacketDirections = new boolean[sequence.size()]; + for (int i = 0; i < sequence.size(); i++) { + if (i == 0) { + // No previous packet; boolean parameter is ignored in this special case. + mPacketDirections[i] = getPacketDirection(null, true, sequence.get(i)); + } else { + // Base direction marker on direction of previous packet. + PcapPacket prevPkt = mSequence.get(i-1); + boolean prevPktDirection = mPacketDirections[i-1]; + mPacketDirections[i] = getPacketDirection(prevPkt, prevPktDirection, sequence.get(i)); + } + } + } + + /** + * Attempt to advance this {@code Layer2SequenceMatcher} by matching {@code packet} against the packet that this + * {@code Layer2SequenceMatcher} expects as the next packet of the sequence it is searching for. + * @param packet + * @return {@code true} if this {@code Layer2SequenceMatcher} could advance by adding {@code packet} to its set of + * matched packets, {@code false} otherwise. + */ + public boolean matchPacket(PcapPacket packet) { + if (getMatchedPacketsCount() == getTargetSequencePacketCount()) { + // We already matched the entire sequence, so we can't match any more packets. + return false; + } + + // Verify that new packet pertains to same flow as previously matched packets, if any. + if (getMatchedPacketsCount() > 0) { + MacAddress pktSrc = PcapPacketUtils.getEthSrcAddr(packet); + MacAddress pktDst = PcapPacketUtils.getEthDstAddr(packet); + MacAddress earlierPktSrc = PcapPacketUtils.getEthSrcAddr(mMatchedPackets.get(0)); + MacAddress earlierPktDst = PcapPacketUtils.getEthDstAddr(mMatchedPackets.get(0)); + if (!(pktSrc.equals(earlierPktSrc) && pktDst.equals(earlierPktDst) || + pktSrc.equals(earlierPktDst) && pktDst.equals(earlierPktSrc))) { + return false; + } + } + + // Get representative of the packet we expect to match next. + PcapPacket expected = mSequence.get(mMatchedPackets.size()); + // First verify if the received packet has the length we're looking for. + if (packet.getOriginalLength() == expected.getOriginalLength()) { + // If this is the first packet, we only need to verify that its length is correct. Time constraints are + // obviously satisfied as there are no previous packets. Furthermore, direction matches by definition as we + // don't know the MAC of the device (or phone) in advance, so we can't enforce a rule saying "first packet + // must originate from this particular MAC". + if (getMatchedPacketsCount() == 0) { + // Store packet as matched and advance. + mMatchedPackets.add(packet); + return true; + } + // Check if direction of packet matches expected direction. + boolean actualDirection = getPacketDirection(mMatchedPackets.get(getMatchedPacketsCount()-1), + mPacketDirections[getMatchedPacketsCount()-1], packet); + boolean expectedDirection = mPacketDirections[getMatchedPacketsCount()]; + if (actualDirection != expectedDirection) { + return false; + } + // Next apply timing constraints: + // 1: to be a match, the packet must have a later timestamp than any other packet currently matched + // 2: does adding the packet cause the max allowed time between first packet and last packet to be exceeded? + if (!packet.getTimestamp().isAfter(mMatchedPackets.get(getMatchedPacketsCount()-1).getTimestamp())) { + return false; + } + if (packet.getTimestamp().isAfter(mMatchedPackets.get(0).getTimestamp(). + plusMillis(TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS))) { + return false; + } + // If we made it here, it means that this packet has the expected length, direction, and obeys the timing + // constraints, so we store it and advance. + mMatchedPackets.add(packet); + if (mMatchedPackets.size() == mSequence.size()) { + // TODO report (to observers?) that we are done? + } + return true; + } + return false; + } + + public int getMatchedPacketsCount() { + return mMatchedPackets.size(); + } + + public int getTargetSequencePacketCount() { + return mSequence.size(); + } + + public List getTargetSequence() { + return mSequence; + } + + public List getMatchedPackets() { + return mMatchedPackets; + } + + /** + * Compute the direction of a packet based on the previous packet. If no previous packet is provided, the direction + * of {@code currPkt} is {@code true} by definition. + * @param prevPkt The previous packet, if any. + * @param prevPktDirection The computed direction of the previous packet + * @param currPkt The current packet for which the direction is to be determined. + * @return The direction of {@code currPkt}. + */ + private boolean getPacketDirection(PcapPacket prevPkt, boolean prevPktDirection, PcapPacket currPkt) { + if (prevPkt == null) { + // By definition, use true as direction marker for first packet + return true; + } + if (PcapPacketUtils.getEthSrcAddr(prevPkt).equals(PcapPacketUtils.getEthSrcAddr(currPkt))) { + // Current packet goes in same direction as previous packet. + return prevPktDirection; + } else { + // Current packet goes in opposite direction of previous packet. + return !prevPktDirection; + } + } + + +} diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer3/Layer3ClusterMatcher.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer3/Layer3ClusterMatcher.java new file mode 100644 index 0000000..f0e3bb6 --- /dev/null +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer3/Layer3ClusterMatcher.java @@ -0,0 +1,353 @@ +package edu.uci.iotproject.detection.layer3; + +import edu.uci.iotproject.detection.AbstractClusterMatcher; +import edu.uci.iotproject.trafficreassembly.layer3.Conversation; +import edu.uci.iotproject.trafficreassembly.layer3.TcpReassembler; +import edu.uci.iotproject.analysis.TcpConversationUtils; +import edu.uci.iotproject.io.PcapHandleReader; +import edu.uci.iotproject.util.PrintUtils; +import org.pcap4j.core.*; + +import java.time.ZoneId; +import java.util.*; +import java.util.stream.Collectors; + +import static edu.uci.iotproject.util.PcapPacketUtils.*; + +/** + * Searches a traffic trace for sequences of packets "belong to" a given cluster (in other words, attempts to classify + * traffic as pertaining to a given cluster). + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class Layer3ClusterMatcher extends AbstractClusterMatcher implements PacketListener { + + // Test client + public static void main(String[] args) throws PcapNativeException, NotOpenException { + +// String path = "/scratch/July-2018"; // Rahmadi + String path = "/Users/varmarken/temp/UCI IoT Project/experiments"; // Janus + final String inputPcapFile = path + "/2018-07/dlink/dlink.wlan1.local.pcap"; + final String signatureFile = path + "/2018-07/dlink/offSignature1.sig"; + + List> signature = PrintUtils.deserializeClustersFromFile(signatureFile); + Layer3ClusterMatcher clusterMatcher = new Layer3ClusterMatcher(signature, null, + (sig, match) -> System.out.println( + String.format("[ !!! SIGNATURE DETECTED AT %s !!! ]", + match.get(0).getTimestamp().atZone(ZoneId.of("America/Los_Angeles"))) + ) + ); + + PcapHandle handle; + try { + handle = Pcaps.openOffline(inputPcapFile, PcapHandle.TimestampPrecision.NANO); + } catch (PcapNativeException pne) { + handle = Pcaps.openOffline(inputPcapFile); + } + PcapHandleReader reader = new PcapHandleReader(handle, p -> true, clusterMatcher); + reader.readFromHandle(); + clusterMatcher.performDetection(); + } + + /** + * The ordered directions of packets in the sequences that make up {@link #mCluster}. + */ + private final Conversation.Direction[] mClusterMemberDirections; + + /** + * For reassembling the observed traffic into TCP connections. + */ + private final TcpReassembler mTcpReassembler = new TcpReassembler(); + + /** + * IP of the router's WAN port (if analyzed traffic is captured at the ISP's point of view). + */ + private final String mRouterWanIp; + + private final ClusterMatchObserver[] mObservers; + + /** + * Create a {@link Layer3ClusterMatcher}. + * @param cluster The cluster that traffic is matched against. + * @param routerWanIp The router's WAN IP if examining traffic captured at the ISP's point of view (used for + * determining the direction of packets). + * @param detectionObservers Client code that wants to get notified whenever the {@link Layer3ClusterMatcher} detects that + * (a subset of) the examined traffic is similar to the traffic that makes up + * {@code cluster}, i.e., when the examined traffic is classified as pertaining to + * {@code cluster}. + */ + public Layer3ClusterMatcher(List> cluster, String routerWanIp, ClusterMatchObserver... detectionObservers) { + super(cluster); + // ===================== PRECONDITION SECTION ===================== + mObservers = Objects.requireNonNull(detectionObservers, "detectionObservers cannot be null"); + if (mObservers.length == 0) { + throw new IllegalArgumentException("no detectionObservers provided"); + } + // Build the cluster members' direction sequence. + // Note: assumes that the provided cluster was captured within the local network (routerWanIp is set to null). + mClusterMemberDirections = getPacketDirections(cluster.get(0), null); + /* + * Enforce restriction on cluster members: all representatives must exhibit the same direction pattern and + * contain the same number of packets. Note that this is a somewhat heavy operation, so it may be disabled later + * on in favor of performance. However, it is only run once (at instantiation), so the overhead may be warranted + * in order to ensure correctness, especially during the development/debugging phase. + */ + if (mCluster.stream(). + anyMatch(inner -> !Arrays.equals(mClusterMemberDirections, getPacketDirections(inner, null)))) { + throw new IllegalArgumentException( + "cluster members must contain the same number of packets and exhibit the same packet direction " + + "pattern" + ); + } + // ================================================================ + mRouterWanIp = routerWanIp; + } + + @Override + public void gotPacket(PcapPacket packet) { + // Present packet to TCP reassembler so that it can be mapped to a connection (if it is a TCP packet). + mTcpReassembler.gotPacket(packet); + } + + /** + * Get the cluster that describes the packet sequence that this {@link Layer3ClusterMatcher} is searching for. + * @return the cluster that describes the packet sequence that this {@link Layer3ClusterMatcher} is searching for. + */ + public List> getCluster() { + return mCluster; + } + + public void performDetection() { + /* + * Let's start out simple by building a version that only works for signatures that do not span across multiple + * TCP conversations... + */ + for (Conversation c : mTcpReassembler.getTcpConversations()) { + if (c.isTls() && c.getTlsApplicationDataPackets().isEmpty() || !c.isTls() && c.getPackets().isEmpty()) { + // Skip empty conversations. + continue; + } + for (List signatureSequence : mCluster) { + if (isTlsSequence(signatureSequence) != c.isTls()) { + // We consider it a mismatch if one is a TLS application data sequence and the other is not. + continue; + } + // Fetch set of packets to examine based on TLS or not. + List cPkts = c.isTls() ? c.getTlsApplicationDataPackets() : c.getPackets(); + /* + * Note: we embed the attempt to detect the signature sequence in a loop in order to capture those cases + * where the same signature sequence appears multiple times in one Conversation. + * + * Note: since we expect all sequences that together make up the signature to exhibit the same direction + * pattern, we can simply pass the precomputed direction array for the signature sequence so that it + * won't have to be recomputed internally in each call to findSubsequenceInSequence(). + */ + Optional> match; + while ((match = findSubsequenceInSequence(signatureSequence, cPkts, mClusterMemberDirections, null)). + isPresent()) { + List matchSeq = match.get(); + // Notify observers about the match. + Arrays.stream(mObservers).forEach(o -> o.onMatch(Layer3ClusterMatcher.this, matchSeq)); + /* + * Get the index in cPkts of the last packet in the sequence of packets that matches the searched + * signature sequence. + */ + int matchSeqEndIdx = cPkts.indexOf(matchSeq.get(matchSeq.size()-1)); + // We restart the search for the signature sequence immediately after that index, so truncate cPkts. + cPkts = cPkts.stream().skip(matchSeqEndIdx + 1).collect(Collectors.toList()); + } + } + /* + * TODO: + * if no item in cluster matches, also perform a distance-based matching to cover those cases where we did + * not manage to capture every single mutation of the sequence during training. + * + * Need to compute average/centroid of cluster to do so...? Compute within-cluster variance, then check if + * distance between input conversation and cluster average/centroid is smaller than or equal to the computed + * variance? + */ + } + } + + /** + * Checks if {@code sequence} is a sequence of TLS packets. Note: the current implementation relies on inspection + * of the port numbers when deciding between TLS vs. non-TLS. Therefore, only the first packet of {@code sequence} + * is examined as it is assumed that all packets in {@code sequence} pertain to the same {@link Conversation} and + * hence share the same set of two src/dst port numbers (albeit possibly alternating between which one is the src + * and which one is the dst, as packets in {@code sequence} may be in alternating directions). + * @param sequence The sequence of packets for which it is to be determined if it is a sequence of TLS packets or + * non-TLS packets. + * @return {@code true} if {@code sequence} is a sequence of TLS packets, {@code false} otherwise. + */ + private boolean isTlsSequence(List sequence) { + // NOTE: Assumes ALL packets in sequence pertain to the same TCP connection! + PcapPacket firstPkt = sequence.get(0); + int srcPort = getSourcePort(firstPkt); + int dstPort = getDestinationPort(firstPkt); + return TcpConversationUtils.isTlsPort(srcPort) || TcpConversationUtils.isTlsPort(dstPort); + } + + /** + * Examine if a given sequence of packets ({@code sequence}) contains a given shorter sequence of packets + * ({@code subsequence}). Note: the current implementation actually searches for a substring as it does not allow + * for interleaving packets in {@code sequence} that are not in {@code subsequence}; for example, if + * {@code subsequence} consists of packet lengths [2, 3, 5] and {@code sequence} consists of packet lengths + * [2, 3, 4, 5], the result will be that there is no match (because of the interleaving 4). If we are to allow + * interleaving packets, we need a modified version of + * this. + * + * @param subsequence The sequence to search for. + * @param sequence The sequence to search. + * @param subsequenceDirections The directions of packets in {@code subsequence} such that for all {@code i}, + * {@code subsequenceDirections[i]} is the direction of the packet returned by + * {@code subsequence.get(i)}. May be set to {@code null}, in which this call will + * internally compute the packet directions. + * @param sequenceDirections The directions of packets in {@code sequence} such that for all {@code i}, + * {@code sequenceDirections[i]} is the direction of the packet returned by + * {@code sequence.get(i)}. May be set to {@code null}, in which this call will internally + * compute the packet directions. + * + * @return An {@link Optional} containing the part of {@code sequence} that matches {@code subsequence}, or an empty + * {@link Optional} if no part of {@code sequence} matches {@code subsequence}. + */ + private Optional> findSubsequenceInSequence(List subsequence, + List sequence, + Conversation.Direction[] subsequenceDirections, + Conversation.Direction[] sequenceDirections) { + if (sequence.size() < subsequence.size()) { + // If subsequence is longer, it cannot be contained in sequence. + return Optional.empty(); + } + if (isTlsSequence(subsequence) != isTlsSequence(sequence)) { + // We consider it a mismatch if one is a TLS application data sequence and the other is not. + return Optional.empty(); + } + // If packet directions have not been precomputed by calling code, we need to construct them. + if (subsequenceDirections == null) { + subsequenceDirections = getPacketDirections(subsequence, mRouterWanIp); + } + if (sequenceDirections == null) { + sequenceDirections = getPacketDirections(sequence, mRouterWanIp); + } + int subseqIdx = 0; + int seqIdx = 0; + while (seqIdx < sequence.size()) { + PcapPacket subseqPkt = subsequence.get(subseqIdx); + PcapPacket seqPkt = sequence.get(seqIdx); + // We only have a match if packet lengths and directions match. + if (subseqPkt.getOriginalLength() == seqPkt.getOriginalLength() && + subsequenceDirections[subseqIdx] == sequenceDirections[seqIdx]) { + // A match; advance both indices to consider next packet in subsequence vs. next packet in sequence. + subseqIdx++; + seqIdx++; + if (subseqIdx == subsequence.size()) { + // We managed to match the entire subsequence in sequence. + // Return the sublist of sequence that matches subsequence. + /* + * TODO: + * ASSUMES THE BACKING LIST (i.e., 'sequence') IS _NOT_ STRUCTURALLY MODIFIED, hence may not work + * for live traces! + */ + return Optional.of(sequence.subList(seqIdx - subsequence.size(), seqIdx)); + } + } else { + // Mismatch. + if (subseqIdx > 0) { + /* + * If we managed to match parts of subsequence, we restart the search for subsequence in sequence at + * the index of sequence where the current mismatch occurred. I.e., we must reset subseqIdx, but + * leave seqIdx untouched. + */ + subseqIdx = 0; + } else { + /* + * First packet of subsequence didn't match packet at seqIdx of sequence, so we move forward in + * sequence, i.e., we continue the search for subsequence in sequence starting at index seqIdx+1 of + * sequence. + */ + seqIdx++; + } + } + } + return Optional.empty(); + } + + /** + * Given a cluster, produces a pruned version of that cluster. In the pruned version, there are no duplicate cluster + * members. Two cluster members are considered identical if their packets lengths and packet directions are + * identical. The resulting pruned cluster is unmodifiable (this applies to both the outermost list as well as the + * nested lists) in order to preserve its integrity when exposed to external code (e.g., through + * {@link #getCluster()}). + * + * @param cluster A cluster to prune. + * @return The resulting pruned cluster. + */ + @Override + protected List> pruneCluster(List> cluster) { + List> prunedCluster = new ArrayList<>(); + for (List originalClusterSeq : cluster) { + boolean alreadyPresent = false; + for (List prunedClusterSeq : prunedCluster) { + Optional> duplicate = findSubsequenceInSequence(originalClusterSeq, prunedClusterSeq, + mClusterMemberDirections, mClusterMemberDirections); + if (duplicate.isPresent()) { + alreadyPresent = true; + break; + } + } + if (!alreadyPresent) { + prunedCluster.add(Collections.unmodifiableList(originalClusterSeq)); + } + } + return Collections.unmodifiableList(prunedCluster); + } + + /** + * Given a {@code List}, generate a {@code Conversation.Direction[]} such that each entry in the + * resulting {@code Conversation.Direction[]} specifies the direction of the {@link PcapPacket} at the corresponding + * index in the input list. + * @param packets The list of packets for which to construct a corresponding array of packet directions. + * @param routerWanIp The IP of the router's WAN port. This is used for determining the direction of packets when + * the traffic is captured just outside the local network (at the ISP side of the router). Set to + * {@code null} if {@code packets} stem from traffic captured within the local network. + * @return A {@code Conversation.Direction[]} specifying the direction of the {@link PcapPacket} at the + * corresponding index in {@code packets}. + */ + private static Conversation.Direction[] getPacketDirections(List packets, String routerWanIp) { + Conversation.Direction[] directions = new Conversation.Direction[packets.size()]; + for (int i = 0; i < packets.size(); i++) { + PcapPacket pkt = packets.get(i); + if (getSourceIp(pkt).equals(getDestinationIp(pkt))) { + // Sanity check: we shouldn't be processing loopback traffic + throw new AssertionError("loopback traffic detected"); + } + if (isSrcIpLocal(pkt) || getSourceIp(pkt).equals(routerWanIp)) { + directions[i] = Conversation.Direction.CLIENT_TO_SERVER; + } else if (isDstIpLocal(pkt) || getDestinationIp(pkt).equals(routerWanIp)) { + directions[i] = Conversation.Direction.SERVER_TO_CLIENT; + } else { + //throw new IllegalArgumentException("no local IP or router WAN port IP found, can't detect direction"); + } + } + return directions; + } + + /** + * Interface used by client code to register for receiving a notification whenever the {@link Layer3ClusterMatcher} + * detects traffic that is similar to the traffic that makes up the cluster returned by + * {@link Layer3ClusterMatcher#getCluster()}. + */ + interface ClusterMatchObserver { + /** + * Callback that is invoked whenever a sequence that is similar to a sequence associated with the cluster (i.e., + * a sequence is a member of the cluster) is detected in the traffic that the associated {@link Layer3ClusterMatcher} + * observes. + * @param clusterMatcher The {@link Layer3ClusterMatcher} that detected a match (classified traffic as pertaining to + * its associated cluster). + * @param match The traffic that was deemed to match the cluster associated with {@code clusterMatcher}. + */ + void onMatch(Layer3ClusterMatcher clusterMatcher, List match); + } + +} diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer3/SignatureDetector.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer3/SignatureDetector.java new file mode 100644 index 0000000..c1a5a9c --- /dev/null +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/detection/layer3/SignatureDetector.java @@ -0,0 +1,660 @@ +package edu.uci.iotproject.detection.layer3; + +import edu.uci.iotproject.analysis.TriggerTrafficExtractor; +import edu.uci.iotproject.analysis.UserAction; +import edu.uci.iotproject.io.PcapHandleReader; +import edu.uci.iotproject.util.PrintUtils; +import org.jgrapht.GraphPath; +import org.jgrapht.alg.shortestpath.DijkstraShortestPath; +import org.jgrapht.graph.DefaultWeightedEdge; +import org.jgrapht.graph.SimpleDirectedWeightedGraph; +import org.pcap4j.core.*; + +import java.time.Duration; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.time.format.FormatStyle; +import java.util.*; +import java.util.function.Consumer; + +/** + * Detects an event signature that spans one or multiple TCP connections. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class SignatureDetector implements PacketListener, Layer3ClusterMatcher.ClusterMatchObserver { + + // Test client + public static void main(String[] args) throws PcapNativeException, NotOpenException { +// if (args.length < 3) { +// String errMsg = String.format("Usage: %s inputPcapFile onSignatureFile offSignatureFile", +// SignatureDetector.class.getSimpleName()); +// System.out.println(errMsg); +// return; +// } +// final String inputPcapFile = args[0]; +// final String onSignatureFile = args[1]; +// final String offSignatureFile = args[2]; + + String path = "/scratch/July-2018"; // Rahmadi +// String path = "/Users/varmarken/temp/UCI IoT Project/experiments"; // Janus +// String path = "/home/jvarmark/iot_project/datasets"; // Hera (server) +// String path = "/raid/varmarken/iot_project/datasets"; // Zeus (server) + + // No activity test + //final String inputPcapFile = path + "/evaluation/no-activity/no-activity.wlan1.pcap"; + + // D-Link Siren experiment +// final String inputPcapFile = path + "/evaluation/dlink-siren/dlink-siren.data.wlan1.pcap"; +// final String inputPcapFile = path + "/evaluation/dlink-siren/dlink-siren.eth0.local.pcap"; + // D-Link Siren DEVICE signatures +// final String onSignatureFile = path + "/2018-08/dlink-siren/onSignature-DLink-Siren-device.sig"; +// final String offSignatureFile = path + "/2018-08/dlink-siren/offSignature-DLink-Siren-device.sig"; + // D-Link Siren PHONE signatures +// final String onSignatureFile = path + "/2018-08/dlink-siren/onSignature-DLink-Siren-phone.sig"; +// final String offSignatureFile = path + "/2018-08/dlink-siren/offSignature-DLink-Siren-phone.sig"; + // TODO: EXPERIMENT - November 19, 2018 + // Hue Bulb experiment +// final String inputPcapFile = path + "/2018-08/hue-bulb/hue-bulb.wlan1.local.pcap"; + // Hue Bulb PHONE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/hue-bulb/signatures/hue-bulb-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/hue-bulb/signatures/hue-bulb-offSignature-phone-side.sig"; + + /* + // Kwikset Doorlock Sep 12 experiment +// final String inputPcapFile = path + "/evaluation/kwikset-doorlock/kwikset-doorlock.data.wlan1.pcap"; + final String inputPcapFile = path + "/evaluation/kwikset-doorlock/kwikset-doorlock.data.eth0.pcap"; +// // Kwikset Doorlock PHONE signatures + final String onSignatureFile = path + "/2018-08/kwikset-doorlock/onSignature-Kwikset-Doorlock-phone-new.sig"; + final String offSignatureFile = path + "/2018-08/kwikset-doorlock/offSignature-Kwikset-Doorlock-phone-new.sig"; + */ + + // D-Link Plug experiment + //final String inputPcapFile = path + "/evaluation/dlink/dlink-plug.data.wlan1.pcap"; +// final String inputPcapFile = path + "/evaluation/dlink/dlink-plug.data.eth0.pcap"; + + // D-Link Plug DEVICE signatures +// final String onSignatureFile = path + "/2018-07/dlink/onSignature-DLink-Plug-device.sig"; +// final String offSignatureFile = path + "/2018-07/dlink/offSignature-DLink-Plug-device.sig"; + // D-Link Plug PHONE signatures +// final String onSignatureFile = path + "/2018-07/dlink/onSignature-DLink-Plug-phone.sig"; +// final String offSignatureFile = path + "/2018-07/dlink/offSignature-DLink-Plug-phone.sig"; + + // TODO: The following are negative tests against the PCAP file from UNSW +// final String inputPcapFile = path + "/UNSW/16-10-04.pcap"; // TODO: Seems to be broken! Zero-payload! +// final String inputPcapFile = path + "/UNSW/16-10-12.pcap"; + +// final String inputPcapFile = path + "/UNSW/16-09-28.pcap"; // TODO: Seems to be broken! Zero-payload! +// final String inputPcapFile = path + "/UNSW/16-10-02.pcap"; // TODO: Seems to be broken! +// final String inputPcapFile = path + "/UNSW/16-10-03.pcap"; // TODO: Seems to be broken! +// final String inputPcapFile = path + "/UNSW/16-10-04-a.pcap"; // TODO: Seems to be broken! Zero-payload! +// final String inputPcapFile = path + "/UNSW/16-10-04-b.pcap"; // TODO: Seems to be broken! Zero-payload! +// final String inputPcapFile = path + "/UNSW/16-10-07.pcap"; // TODO: Seems to be broken! +// final String inputPcapFile = path + "/UNSW/16-10-08.pcap"; // TODO: Seems to be broken! +// final String inputPcapFile = path + "/UNSW/16-10-09.pcap"; // TODO: Seems to be broken! +// final String inputPcapFile = path + "/UNSW/16-10-10.pcap"; // TODO: Seems to be broken! +// final String inputPcapFile = path + "/UNSW/16-10-11.pcap"; // TODO: Seems to be broken! + // TODO: The following one is very long!!! - Split into smaller files! +// final String inputPcapFile = path + "/UNSW/16-10-12-a.pcap"; +// final String inputPcapFile = path + "/UNSW/16-10-12-b.pcap"; +// final String inputPcapFile = path + "/UNSW/16-10-12-c.pcap"; +// final String inputPcapFile = path + "/UNSW/16-10-12-d.pcap"; + +// final String inputPcapFile = path + "/UNSW/16-09-23.pcap"; +// final String inputPcapFile = path + "/UNSW/16-09-24.pcap"; +// final String inputPcapFile = path + "/UNSW/16-09-25.pcap"; +// final String inputPcapFile = path + "/UNSW/16-09-26.pcap"; +// final String inputPcapFile = path + "/UNSW/16-09-27.pcap"; +// final String inputPcapFile = path + "/UNSW/16-09-29.pcap"; +// final String inputPcapFile = path + "/UNSW/16-10-01.pcap"; +// final String inputPcapFile = path + "/UNSW/16-10-06.pcap"; + // Negative test: dataset from UNB +// final String inputPcapFile = path + "/evaluation/negative-datasets/UNB/Monday-WorkingHours_one-local-endpoint.pcap"; + + // TODO: The following are tests for signatures against training data + + // D-Link Plug experiment +// final String inputPcapFile = path + "/training/dlink-plug/wlan1/dlink-plug.wlan1.local.pcap"; + // D-Link Plug DEVICE signatures +// final String onSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-onSignature-device-side.sig"; +// final String offSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-offSignature-device-side.sig"; + // D-Link Plug PHONE signatures +// final String onSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/training/dlink-plug/signatures/dlink-plug-offSignature-phone-side.sig"; + + // TODO: EXPERIMENT - November 7, 2018 + // D-Link Plug experiment + //final String inputPcapFile = path + "/experimental_result/standalone/dlink-plug/wlan1/dlink-plug.wlan1.local.pcap"; + //final String inputPcapFile = path + "/experimental_result/smarthome/dlink-plug/wlan1/dlink-plug.wlan1.detection.pcap"; + //final String inputPcapFile = path + "/experimental_result/smarthome/dlink-plug/eth0/dlink-plug.eth0.detection.pcap"; + // D-Link Plug DEVICE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-onSignature-device-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-offSignature-device-side.sig"; + // D-Link Plug PHONE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/dlink-plug/signatures/dlink-plug-offSignature-phone-side.sig"; + + // TODO: EXPERIMENT - November 9, 2018 + // D-Link Siren experiment + //final String inputPcapFile = path + "/experimental_result/standalone/dlink-siren/wlan1/dlink-siren.wlan1.local.pcap"; + //final String inputPcapFile = path + "/experimental_result/smarthome/dlink-siren/wlan1/dlink-siren.wlan1.detection.pcap"; +// final String inputPcapFile = path + "/experimental_result/smarthome/dlink-siren/eth0/dlink-siren.eth0.detection.pcap"; + // D-Link Siren DEVICE signatures + // TODO: The device signature does not have pairs---only one packet which is 216, so we don't consider this as a signature +// final String onSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-onSignature-device-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-offSignature-device-side.sig"; + // D-Link Siren PHONE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/dlink-siren/signatures/dlink-siren-offSignature-phone-side.sig"; +// final String onSignatureFile = path + "/training/signatures/dlink-siren/dlink-siren-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/training/signatures/dlink-siren/dlink-siren-offSignature-phone-side.sig"; + + // TP-Link Plug experiment +//// final String inputPcapFile = path + "/training/tplink-plug/wlan1/tplink-plug.wlan1.local.pcap"; +//// final String inputPcapFile = path + "/experimental_result/wifi-Sniffer/tests2/airtool_2019-01-04_11.08.45.AM.pcap"; +// final String inputPcapFile = path + "/experimental_result/wifi-Sniffer/tests2/command-frames-only.pcap"; +// // TP-Link Plug DEVICE signatures +// final String onSignatureFile = path + "/training/tplink-plug/signatures/tplink-plug-onSignature-device-side.sig"; +// final String offSignatureFile = path + "/training/tplink-plug/signatures/tplink-plug-offSignature-device-side.sig"; + // TODO: EXPERIMENT - November 8, 2018 + // TP-Link Plug experiment +// final String inputPcapFile = path + "/experimental_result/standalone/tplink-plug/wlan1/tplink-plug.wlan1.local.pcap"; +// final String inputPcapFile = path + "/experimental_result/standalone/tplink-plug/eth0/tplink-plug.eth0.local.pcap"; +// final String inputPcapFile = path + "/experimental_result/smarthome/tplink-plug/wlan1/tplink-plug.wlan1.detection.pcap"; + //final String inputPcapFile = path + "/experimental_result/smarthome/tplink-plug/eth0/tplink-plug.eth0.detection.pcap"; + // TP-Link Plug DEVICE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-device-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-device-side.sig"; +// final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-device-side-outbound.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-device-side-outbound.sig"; + // TP-Link Plug PHONE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/tplink-plug/signatures/tplink-plug-offSignature-phone-side.sig"; + + // Arlo camera experiment +// final String inputPcapFile = path + "/training/arlo-camera/wlan1/arlo-camera.wlan1.local.pcap"; +//// // TP-Link Plug DEVICE signatures +// final String onSignatureFile = path + "/training/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/training/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig"; + // TODO: EXPERIMENT - November 13, 2018 + // Arlo Camera experiment +// final String inputPcapFile = path + "/experimental_result/standalone/arlo-camera/wlan1/arlo-camera.wlan1.local.pcap"; +// final String inputPcapFile = path + "/experimental_result/standalone/arlo-camera/eth0/arlo-camera.eth0.local.pcap"; +// final String inputPcapFile = path + "/experimental_result/smarthome/arlo-camera/wlan1/arlo-camera.wlan1.detection.pcap"; +// final String inputPcapFile = path + "/experimental_result/smarthome/arlo-camera/eth0/arlo-camera.eth0.detection.pcap"; +// final String inputPcapFile = path + "/training/arlo-camera/eth0/arlo-camera.eth0.local.pcap"; + // Arlo Camera PHONE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/arlo-camera/signatures/arlo-camera-offSignature-phone-side.sig"; + + // Amazon Alexa experiment +// final String inputPcapFile = path + "/training/amazon-alexa/wlan1/alexa2.wlan1.local.pcap"; +// // TP-Link Plug DEVICE signatures +// final String onSignatureFile = path + "/training/amazon-alexa/signatures/amazon-alexa-onSignature-device-side.sig"; +// final String offSignatureFile = path + "/training/amazon-alexa/signatures/amazon-alexa-offSignature-device-side.sig"; + + // SmartThings Plug experiment +// final String inputPcapFile = path + "/training/st-plug/wlan1/st-plug.wlan1.local.pcap"; +// // SmartThings Plug DEVICE signatures +// //final String onSignatureFile = path + "/training/st-plug/signatures/st-plug-onSignature-device-side.sig"; +// //final String offSignatureFile = path + "/training/st-plug/signatures/st-plug-offSignature-device-side.sig"; +// // SmartThings Plug PHONE signatures +// final String onSignatureFile = path + "/training/st-plug/signatures/st-plug-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/training/st-plug/signatures/st-plug-offSignature-phone-side.sig"; + // TODO: EXPERIMENT - November 12, 2018 + // SmartThings Plug experiment +// final String inputPcapFile = path + "/experimental_result/standalone/st-plug/wlan1/st-plug.wlan1.local.pcap"; +// final String inputPcapFile = path + "/experimental_result/standalone/st-plug/eth0/st-plug.eth0.local.pcap"; +// //final String inputPcapFile = path + "/experimental_result/smarthome/st-plug/wlan1/st-plug.wlan1.detection.pcap"; +// final String inputPcapFile = path + "/experimental_result/smarthome/st-plug/eth0/st-plug.eth0.detection.pcap"; +// // SmartThings Plug PHONE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/st-plug/signatures/st-plug-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/st-plug/signatures/st-plug-offSignature-phone-side.sig"; +// final String onSignatureFile = path + "/training/signatures/st-plug/st-plug-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/training/signatures/st-plug/st-plug-offSignature-phone-side.sig"; + + // TODO: EXPERIMENT - January 9, 2018 + // Blossom Sprinkler experiment +// final String inputPcapFile = path + "/experimental_result/standalone/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.local.pcap"; + final String inputPcapFile = path + "/experimental_result/smarthome/blossom-sprinkler/eth0/blossom-sprinkler.eth0.detection.pcap"; +// final String inputPcapFile = path + "/experimental_result/smarthome/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.detection.pcap"; + // Blossom Sprinkler DEVICE signatures + final String onSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig"; + final String offSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig"; + // Blossom Sprinkler PHONE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-phone-side.sig"; + + // LiFX Bulb experiment +// final String inputPcapFile = path + "/training/lifx-bulb/wlan1/lifx-bulb.wlan1.local.pcap"; +// // LiFX Bulb DEVICE signatures +// final String onSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-onSignature-device-side.sig"; +// final String offSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-offSignature-device-side.sig"; + // LiFX Bulb PHONE signatures +// final String onSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/training/lifx-bulb/signatures/lifx-bulb-offSignature-phone-side.sig"; + + // Blossom Sprinkler experiment +// //final String inputPcapFile = path + "/training/blossom-sprinkler/wlan1/blossom-sprinkler.wlan1.local.pcap"; +// final String inputPcapFile = path + "/2018-08/blossom/blossom.wlan1.local.pcap"; +// //final String inputPcapFile = path + "/training/blossom-sprinkler/eth0/blossom-sprinkler.eth0.local.pcap"; +// // Blossom Sprinkler DEVICE signatures +// final String onSignatureFile = path + "/training/blossom-sprinkler/signatures/blossom-sprinkler-onSignature-device-side.sig"; +// final String offSignatureFile = path + "/training/blossom-sprinkler/signatures/blossom-sprinkler-offSignature-device-side.sig"; + + // Nest Thermostat experiment +// final String inputPcapFile = path + "/training/nest-thermostat/wlan1/nest-thermostat.wlan1.local.pcap"; +// // Nest Thermostat DEVICE signatures +//// final String onSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-onSignature-device-side.sig"; +//// final String offSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-offSignature-device-side.sig"; +// // Nest Thermostat PHONE signatures +// final String onSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/training/nest-thermostat/signatures/nest-thermostat-offSignature-phone-side.sig"; + // TODO: EXPERIMENT - November 15, 2018 + // Nest Thermostat experiment +// final String inputPcapFile = path + "/experimental_result/standalone/nest-thermostat/wlan1/nest-thermostat.wlan1.local.pcap"; +//// final String inputPcapFile = path + "/experimental_result/standalone/nest-thermostat/eth0/nest-thermostat.eth0.local.pcap"; +//// final String inputPcapFile = path + "/experimental_result/smarthome/nest-thermostat/wlan1/nest-thermostat.wlan1.detection.pcap"; +// final String inputPcapFile = path + "/experimental_result/smarthome/nest-thermostat/eth0/nest-thermostat.eth0.detection.pcap"; +//// // Nest Thermostat PHONE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/nest-thermostat/signatures/nest-thermostat-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/nest-thermostat/signatures/nest-thermostat-offSignature-phone-side.sig"; + + /* + // Hue Bulb experiment + final String inputPcapFile = path + "/training/hue-bulb/wlan1/hue-bulb.wlan1.local.pcap"; + // Hue Bulb PHONE signatures + final String onSignatureFile = path + "/training/hue-bulb/signatures/hue-bulb-onSignature-phone-side.sig"; + final String offSignatureFile = path + "/training/hue-bulb/signatures/hue-bulb-offSignature-phone-side.sig"; + */ + + + + // TP-Link Bulb experiment +// final String inputPcapFile = path + "/training/tplink-bulb/wlan1/tplink-bulb.wlan1.local.pcap"; +// // TP-Link Bulb PHONE signatures +// final String onSignatureFile = path + "/training/tplink-bulb/signatures/tplink-bulb-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/training/tplink-bulb/signatures/tplink-bulb-offSignature-phone-side.sig"; + // TODO: EXPERIMENT - November 16, 2018 + // TP-Link Bulb experiment +// final String inputPcapFile = path + "/experimental_result/standalone/tplink-bulb/wlan1/tplink-bulb.wlan1.local.pcap"; +// final String inputPcapFile = path + "/experimental_result/standalone/tplink-bulb/eth0/tplink-bulb.eth0.local.pcap"; +// final String inputPcapFile = path + "/experimental_result/smarthome/tplink-bulb/wlan1/tplink-bulb.wlan1.detection.pcap"; +//// final String inputPcapFile = path + "/experimental_result/smarthome/tplink-bulb/eth0/tplink-bulb.eth0.detection.pcap"; +// // TP-Link Bulb PHONE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/tplink-bulb/signatures/tplink-bulb-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/tplink-bulb/signatures/tplink-bulb-offSignature-phone-side.sig"; + + /* + // WeMo Plug experiment + final String inputPcapFile = path + "/training/wemo-plug/wlan1/wemo-plug.wlan1.local.pcap"; + // WeMo Plug PHONE signatures + final String onSignatureFile = path + "/training/wemo-plug/signatures/wemo-plug-onSignature-device-side.sig"; + final String offSignatureFile = path + "/training/wemo-plug/signatures/wemo-plug-offSignature-device-side.sig"; + */ + // TODO: EXPERIMENT - November 20, 2018 + // WeMo Plug experiment +// final String inputPcapFile = path + "/experimental_result/standalone/wemo-plug/wlan1/wemo-plug.wlan1.local.pcap"; +// final String inputPcapFile = path + "/experimental_result/standalone/wemo-plug/eth0/wemo-plug.eth0.local.pcap"; + // TODO: WE HAVE 4 ADDITIONAL EVENTS (TRIGGERED MANUALLY), SO WE JUST IGNORE THEM BECAUSE THEY HAPPENED BEFORE + // TODO: THE ACTUAL TRIGGERS +// final String inputPcapFile = path + "/experimental_result/smarthome/wemo-plug/wlan1/wemo-plug.wlan1.detection.pcap"; +//// final String inputPcapFile = path + "/experimental_result/smarthome/wemo-plug/eth0/wemo-plug.eth0.detection.pcap"; +// // WeMo Plug PHONE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/wemo-plug/signatures/wemo-plug-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/wemo-plug/signatures/wemo-plug-offSignature-phone-side.sig"; + + /* + // WeMo Insight Plug experiment + final String inputPcapFile = path + "/training/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.local.pcap"; + // WeMo Insight Plug PHONE signatures + final String onSignatureFile = path + "/training/wemo-insight-plug/signatures/wemo-insight-plug-onSignature-device-side.sig"; + final String offSignatureFile = path + "/training/wemo-insight-plug/signatures/wemo-insight-plug-offSignature-device-side.sig"; + */ + // TODO: EXPERIMENT - November 21, 2018 + // WeMo Insight Plug experiment +// final String inputPcapFile = path + "/experimental_result/standalone/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.local.pcap"; +// final String inputPcapFile = path + "/experimental_result/standalone/wemo-insight-plug/eth0/wemo-insight-plug.eth0.local.pcap"; + // TODO: WE HAVE 1 ADDITIONAL EVENT (FROM WEMO PLUG) +// final String inputPcapFile = path + "/experimental_result/smarthome/wemo-insight-plug/wlan1/wemo-insight-plug.wlan1.detection.pcap"; +// final String inputPcapFile = path + "/experimental_result/smarthome/wemo-insight-plug/eth0/wemo-insight-plug.eth0.detection.pcap"; + // WeMo Insight Plug PHONE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/wemo-insight-plug/signatures/wemo-insight-plug-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/wemo-insight-plug/signatures/wemo-insight-plug-offSignature-phone-side.sig"; + + + // Kwikset Doorlock Sep 12 experiment +// final String inputPcapFile = path + "/2018-08/kwikset-doorlock/kwikset3.wlan1.local.pcap"; +// // Kwikset Doorlock PHONE signatures +// final String onSignatureFile = path + "/2018-08/kwikset-doorlock/onSignature-Kwikset-Doorlock-phone.sig"; +// final String offSignatureFile = path + "/2018-08/kwikset-doorlock/offSignature-Kwikset-Doorlock-phone.sig"; + // TODO: EXPERIMENT - November 10, 2018 + // Kwikset Door lock experiment +// final String inputPcapFile = path + "/experimental_result/standalone/kwikset-doorlock/wlan1/kwikset-doorlock.wlan1.local.pcap"; +// //final String inputPcapFile = path + "/experimental_result/smarthome/kwikset-doorlock/wlan1/kwikset-doorlock.wlan1.detection.pcap"; +// final String inputPcapFile = path + "/experimental_result/smarthome/kwikset-doorlock/eth0/kwikset-doorlock.eth0.detection.pcap"; +//// // Kwikset Door lock PHONE signatures +// final String onSignatureFile = path + "/experimental_result/standalone/kwikset-doorlock/signatures/kwikset-doorlock-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/experimental_result/standalone/kwikset-doorlock/signatures/kwikset-doorlock-offSignature-phone-side.sig"; +// final String onSignatureFile = path + "/training/signatures/kwikset-doorlock/kwikset-doorlock-onSignature-phone-side.sig"; +// final String offSignatureFile = path + "/training/signatures/kwikset-doorlock/kwikset-doorlock-offSignature-phone-side.sig"; + + + + // D-Link Siren experiment +// final String inputPcapFile = path + "/2018-08/dlink-siren/dlink-siren.wlan1.local.pcap"; + // D-Link Siren DEVICE signatures + //final String onSignatureFile = path + "/2018-08/dlink-siren/onSignature-DLink-Siren-device.sig"; + //final String offSignatureFile = path + "/2018-08/dlink-siren/offSignature-DLink-Siren-device.sig"; + // D-Link Siren PHONE signatures +// final String onSignatureFile = path + "/2018-08/dlink-siren/onSignature-DLink-Siren-phone.sig"; +// final String offSignatureFile = path + "/2018-08/dlink-siren/offSignature-DLink-Siren-phone.sig"; + + + // Output file names used (to make it easy to catch if one forgets to change them) + System.out.println("ON signature file in use is " + onSignatureFile); + System.out.println("OFF signature file in use is " + offSignatureFile); + System.out.println("PCAP file that is the target of detection is " + inputPcapFile); + + List>> onSignature = PrintUtils.deserializeSignatureFromFile(onSignatureFile); + List>> offSignature = PrintUtils.deserializeSignatureFromFile(offSignatureFile); + + // LAN +// SignatureDetector onDetector = new SignatureDetector(onSignature, null); +// SignatureDetector offDetector = new SignatureDetector(offSignature, null); + // WAN + SignatureDetector onDetector = new SignatureDetector(onSignature, "128.195.205.105"); + SignatureDetector offDetector = new SignatureDetector(offSignature, "128.195.205.105"); + + final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofLocalizedDateTime(FormatStyle.MEDIUM). + withLocale(Locale.US).withZone(ZoneId.of("America/Los_Angeles")); + + // Outputs information about a detected event to std.out + final Consumer outputter = ua -> { + String eventDescription; + switch (ua.getType()) { + case TOGGLE_ON: + eventDescription = "ON"; + break; + case TOGGLE_OFF: + eventDescription = "OFF"; + break; + default: + throw new AssertionError("unhandled event type"); + } + //String output = String.format("[ !!! %s SIGNATURE DETECTED at %s !!! ]", + // eventDescription, dateTimeFormatter.format(ua.getTimestamp())); + String output = String.format("%s", + dateTimeFormatter.format(ua.getTimestamp())); + System.out.println(output); + }; + + // Let's create observers that construct a UserAction representing the detected event. + final List detectedEvents = new ArrayList<>(); + onDetector.addObserver((searched, match) -> { + PcapPacket firstPkt = match.get(0).get(0); + detectedEvents.add(new UserAction(UserAction.Type.TOGGLE_ON, firstPkt.getTimestamp())); + }); + offDetector.addObserver((searched, match) -> { + PcapPacket firstPkt = match.get(0).get(0); + detectedEvents.add(new UserAction(UserAction.Type.TOGGLE_OFF, firstPkt.getTimestamp())); + }); + + PcapHandle handle; + try { + handle = Pcaps.openOffline(inputPcapFile, PcapHandle.TimestampPrecision.NANO); + } catch (PcapNativeException pne) { + handle = Pcaps.openOffline(inputPcapFile); + } + PcapHandleReader reader = new PcapHandleReader(handle, p -> true, onDetector, offDetector); + reader.readFromHandle(); + + // TODO: need a better way of triggering detection than this... + onDetector.mClusterMatchers.forEach(cm -> cm.performDetection()); + offDetector.mClusterMatchers.forEach(cm -> cm.performDetection()); + + // Sort the list of detected events by timestamp to make it easier to compare it line-by-line with the trigger + // times file. + Collections.sort(detectedEvents, Comparator.comparing(UserAction::getTimestamp)); + + // Output the detected events + detectedEvents.forEach(outputter); + + System.out.println("Number of detected events of type " + UserAction.Type.TOGGLE_ON + ": " + + detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_ON).count()); + System.out.println("Number of detected events of type " + UserAction.Type.TOGGLE_OFF + ": " + + detectedEvents.stream().filter(ua -> ua.getType() == UserAction.Type.TOGGLE_OFF).count()); + + + // TODO: Temporary clean up until we clean the pipeline +// List cleanedDetectedEvents = SignatureDetector.removeDuplicates(detectedEvents); +// cleanedDetectedEvents.forEach(outputter); + } + + /** + * The signature that this {@link SignatureDetector} is searching for. + */ + private final List>> mSignature; + + /** + * The {@link Layer3ClusterMatcher}s in charge of detecting each individual sequence of packets that together make up the + * the signature. + */ + private final List mClusterMatchers; + + /** + * For each {@code i} ({@code i >= 0 && i < pendingMatches.length}), {@code pendingMatches[i]} holds the matches + * found by the {@link Layer3ClusterMatcher} at {@code mClusterMatchers.get(i)} that have yet to be "consumed", i.e., + * have yet to be included in a signature detected by this {@link SignatureDetector} (a signature can be encompassed + * of multiple packet sequences occurring shortly after one another on multiple connections). + */ + private final List>[] pendingMatches; + + /** + * Maps a {@link Layer3ClusterMatcher} to its corresponding index in {@link #pendingMatches}. + */ + private final Map mClusterMatcherIds; + + private final List mObservers = new ArrayList<>(); + + /** + * Remove duplicates in {@code List} of {@code UserAction} objects. We need to clean this up for user actions + * that appear multiple times. + * TODO: This static method is probably just for temporary and we could get rid of this after we clean up + * TODO: the pipeline + * + * @param listUserAction A {@link List} of {@code UserAction}. + * + */ + public static List removeDuplicates(List listUserAction) { + + // Iterate and check for duplicates (check timestamps) + Set epochSecondSet = new HashSet<>(); + // Create a target list for cleaned up list + List listUserActionClean = new ArrayList<>(); + for(UserAction userAction : listUserAction) { + // Don't insert if any duplicate is found + if(!epochSecondSet.contains(userAction.getTimestamp().getEpochSecond())) { + listUserActionClean.add(userAction); + epochSecondSet.add(userAction.getTimestamp().getEpochSecond()); + } + } + return listUserActionClean; + } + + public SignatureDetector(List>> searchedSignature, String routerWanIp) { + // note: doesn't protect inner lists from changes :'( + mSignature = Collections.unmodifiableList(searchedSignature); + // Generate corresponding/appropriate ClusterMatchers based on the provided signature + List clusterMatchers = new ArrayList<>(); + for (List> cluster : mSignature) { + clusterMatchers.add(new Layer3ClusterMatcher(cluster, routerWanIp, this)); + } + mClusterMatchers = Collections.unmodifiableList(clusterMatchers); + + // < exploratory > + pendingMatches = new List[mClusterMatchers.size()]; + for (int i = 0; i < pendingMatches.length; i++) { + pendingMatches[i] = new ArrayList<>(); + } + Map clusterMatcherIds = new HashMap<>(); + for (int i = 0; i < mClusterMatchers.size(); i++) { + clusterMatcherIds.put(mClusterMatchers.get(i), i); + } + mClusterMatcherIds = Collections.unmodifiableMap(clusterMatcherIds); + } + + public void addObserver(SignatureDetectionObserver observer) { + mObservers.add(observer); + } + + public boolean removeObserver(SignatureDetectionObserver observer) { + return mObservers.remove(observer); + } + + @Override + public void gotPacket(PcapPacket packet) { + // simply delegate packet reception to all ClusterMatchers. + mClusterMatchers.forEach(cm -> cm.gotPacket(packet)); + } + + @Override + public void onMatch(Layer3ClusterMatcher clusterMatcher, List match) { + // Add the match at the corresponding index + pendingMatches[mClusterMatcherIds.get(clusterMatcher)].add(match); + checkSignatureMatch(); + } + + private void checkSignatureMatch() { + // << Graph-based approach using Balint's idea. >> + // This implementation assumes that the packets in the inner lists (the sequences) are ordered by asc timestamp. + + // There cannot be a signature match until each Layer3ClusterMatcher has found a match of its respective sequence. + if (Arrays.stream(pendingMatches).noneMatch(l -> l.isEmpty())) { + // Construct the DAG + final SimpleDirectedWeightedGraph graph = + new SimpleDirectedWeightedGraph<>(DefaultWeightedEdge.class); + // Add a vertex for each match found by all ClusterMatchers + // And maintain an array to keep track of what cluster matcher each vertex corresponds to + final List[] vertices = new List[pendingMatches.length]; + for (int i = 0; i < pendingMatches.length; i++) { + vertices[i] = new ArrayList<>(); + for (List sequence : pendingMatches[i]) { + Vertex v = new Vertex(sequence); + vertices[i].add(v); // retain reference for later when we are to add edges + graph.addVertex(v); // add to vertex to graph + } + } + // Add dummy source and sink vertices to facilitate search. + final Vertex source = new Vertex(null); + final Vertex sink = new Vertex(null); + graph.addVertex(source); + graph.addVertex(sink); + // The source is connected to all vertices that wrap the sequences detected by Layer3ClusterMatcher at index 0. + // Note: zero cost edges as this is just a dummy link to facilitate search from a common start node. + for (Vertex v : vertices[0]) { + DefaultWeightedEdge edge = graph.addEdge(source, v); + graph.setEdgeWeight(edge, 0.0); + } + // Similarly, all vertices that wrap the sequences detected by the last Layer3ClusterMatcher of the signature + // are connected to the sink node. + for (Vertex v : vertices[vertices.length-1]) { + DefaultWeightedEdge edge = graph.addEdge(v, sink); + graph.setEdgeWeight(edge, 0.0); + } + // Now link sequences detected by Layer3ClusterMatcher at index i to sequences detected by Layer3ClusterMatcher at index + // i+1 if they obey the timestamp constraint (i.e., that the latter is later in time than the former). + for (int i = 0; i < vertices.length; i++) { + int j = i + 1; + if (j < vertices.length) { + for (Vertex iv : vertices[i]) { + PcapPacket ivLast = iv.sequence.get(iv.sequence.size()-1); + for (Vertex jv : vertices[j]) { + PcapPacket jvFirst = jv.sequence.get(jv.sequence.size()-1); + if (ivLast.getTimestamp().isBefore(jvFirst.getTimestamp())) { + DefaultWeightedEdge edge = graph.addEdge(iv, jv); + // The weight is the duration of the i'th sequence plus the duration between the i'th + // and i+1'th sequence. + Duration d = Duration. + between(iv.sequence.get(0).getTimestamp(), jvFirst.getTimestamp()); + // Unfortunately weights are double values, so must convert from long to double. + // TODO: need nano second precision? If so, use d.toNanos(). + // TODO: risk of overflow when converting from long to double..? + graph.setEdgeWeight(edge, Long.valueOf(d.toMillis()).doubleValue()); + } + // Alternative version if we cannot assume that sequences are ordered by timestamp: +// if (iv.sequence.stream().max(Comparator.comparing(PcapPacket::getTimestamp)).get() +// .getTimestamp().isBefore(jv.sequence.stream().min( +// Comparator.comparing(PcapPacket::getTimestamp)).get().getTimestamp())) { +// +// } + } + } + } + } + // Graph construction complete, run shortest-path to find a (potential) signature match. + DijkstraShortestPath dijkstra = new DijkstraShortestPath<>(graph); + GraphPath shortestPath = dijkstra.getPath(source, sink); + if (shortestPath != null) { + // The total weight is the duration between the first packet of the first sequence and the last packet + // of the last sequence, so we simply have to compare the weight against the timeframe that we allow + // the signature to span. For now we just use the inclusion window we defined for training purposes. + // Note however, that we must convert back from double to long as the weight is stored as a double in + // JGraphT's API. + if (((long)shortestPath.getWeight()) < TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS) { + // There's a signature match! + // Extract the match from the vertices + List> signatureMatch = new ArrayList<>(); + for(Vertex v : shortestPath.getVertexList()) { + if (v == source || v == sink) { + // Skip the dummy source and sink nodes. + continue; + } + signatureMatch.add(v.sequence); + // As there is a one-to-one correspondence between vertices[] and pendingMatches[], we know that + // the sequence we've "consumed" for index i of the matched signature is also at index i in + // pendingMatches. We must remove it from pendingMatches so that we don't use it to construct + // another signature match in a later call. + pendingMatches[signatureMatch.size()-1].remove(v.sequence); + } + // Declare success: notify observers + mObservers.forEach(obs -> obs.onSignatureDetected(mSignature, + Collections.unmodifiableList(signatureMatch))); + } + } + } + } + + /** + * Used for registering for notifications of signatures detected by a {@link SignatureDetector}. + */ + interface SignatureDetectionObserver { + + /** + * Invoked when the {@link SignatureDetector} detects the presence of a signature in the traffic that it's + * examining. + * @param searchedSignature The signature that the {@link SignatureDetector} reporting the match is searching + * for. + * @param matchingTraffic The actual traffic trace that matches the searched signature. + */ + void onSignatureDetected(List>> searchedSignature, + List> matchingTraffic); + } + + /** + * Encapsulates a {@code List} so as to allow the list to be used as a vertex in a graph while avoiding + * the expensive {@link AbstractList#equals(Object)} calls when adding vertices to the graph. + * Using this wrapper makes the incurred {@code equals(Object)} calls delegate to {@link Object#equals(Object)} + * instead of {@link AbstractList#equals(Object)}. The net effect is a faster implementation, but the graph will not + * recognize two lists that contain the same items--from a value and not reference point of view--as the same + * vertex. However, this is fine for our purposes -- in fact restricting it to reference equality seems more + * appropriate. + */ + private static class Vertex { + private final List sequence; + private Vertex(List wrappedSequence) { + sequence = wrappedSequence; + } + } +} diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/evaluation/SanitySignatureGenerator.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/evaluation/SanitySignatureGenerator.java index 9cecf56..060387a 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/evaluation/SanitySignatureGenerator.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/evaluation/SanitySignatureGenerator.java @@ -1,7 +1,7 @@ package edu.uci.iotproject.evaluation; -import edu.uci.iotproject.Conversation; -import edu.uci.iotproject.TcpReassembler; +import edu.uci.iotproject.trafficreassembly.layer3.Conversation; +import edu.uci.iotproject.trafficreassembly.layer3.TcpReassembler; import edu.uci.iotproject.io.PcapHandleReader; import edu.uci.iotproject.util.PrintUtils; import org.pcap4j.core.*; diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2Flow.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2Flow.java new file mode 100644 index 0000000..2d804bd --- /dev/null +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2Flow.java @@ -0,0 +1,95 @@ +package edu.uci.iotproject.trafficreassembly.layer2; + +import org.pcap4j.core.PcapPacket; +import org.pcap4j.packet.EthernetPacket; +import org.pcap4j.util.MacAddress; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Models a layer 2 flow: groups packets exchanged between two specific endpoints (MAC addresses). + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class Layer2Flow { + + /** + * The first endpoint of this layer 2 flow. + */ + private final MacAddress mEndpoint1; + + /** + * The second endpoint of this layer 2 flow. + */ + private final MacAddress mEndpoint2; + + /** + * Clients observing for changes to this layer 2 flow. + */ + private final List mFlowObservers = new ArrayList<>(); + + public Layer2Flow(MacAddress endpoint1, MacAddress endpoint2) { + mEndpoint1 = endpoint1; + mEndpoint2 = endpoint2; + } + + /** + * Register as an observer of this flow. + * @param observer The client that is to be notified whenever this flow changes (has new packets added). + */ + public void addFlowObserver(Layer2FlowObserver observer) { + mFlowObservers.add(observer); + } + + /** + * Deregister as an observer of this flow. + * @param observer The client that no longer wishes to be notified whenever this flow changes. + */ + public void removeFlowObserver(Layer2FlowObserver observer) { + mFlowObservers.remove(observer); + } + + /** + * The packets in the flow. + */ + private final List mPackets = new ArrayList<>(); + + /** + * Add a packet to this flow. + * @param packet The packet that is to be added to the flow. + */ + public void addPacket(PcapPacket packet) { + verifyAddresses(packet); + mPackets.add(packet); + // Notify flow observers of the new packet + mFlowObservers.forEach(o -> o.onNewPacket(this, packet)); + } + + /** + * Get the packets pertaining to this flow. + * @return The packets pertaining to this flow. + */ + public List getPackets() { + return Collections.unmodifiableList(mPackets); + } + + /** + * Verify that a packet pertains to this flow. + * @param packet The packet that is to be verified. + */ + private void verifyAddresses(PcapPacket packet) { + EthernetPacket ethPkt = packet.get(EthernetPacket.class); + MacAddress srcAddr = ethPkt.getHeader().getSrcAddr(); + MacAddress dstAddr = ethPkt.getHeader().getDstAddr(); + if ((mEndpoint1.equals(srcAddr) && mEndpoint2.equals(dstAddr)) || + (mEndpoint1.equals(dstAddr) && mEndpoint2.equals(srcAddr))) { + // All is good. + return; + } + throw new IllegalArgumentException("Mismatch in MACs: packet does not pertain to this flow"); + } + +} diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowObserver.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowObserver.java new file mode 100644 index 0000000..e1648ba --- /dev/null +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowObserver.java @@ -0,0 +1,20 @@ +package edu.uci.iotproject.trafficreassembly.layer2; + +import org.pcap4j.core.PcapPacket; + +/** + * Interface for observing a {@link Layer2Flow}. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public interface Layer2FlowObserver { + + /** + * Invoked when a new packet is added to the observed flow. + * @param flow The observed flow. + * @param newPacket The packet that was added to the flow. + */ + void onNewPacket(Layer2Flow flow, PcapPacket newPacket); + +} diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowReassembler.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowReassembler.java new file mode 100644 index 0000000..e7b7304 --- /dev/null +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowReassembler.java @@ -0,0 +1,86 @@ +package edu.uci.iotproject.trafficreassembly.layer2; + +import edu.uci.iotproject.trafficreassembly.layer2.Layer2Flow; +import edu.uci.iotproject.trafficreassembly.layer2.Layer2FlowReassemblerObserver; +import org.pcap4j.core.PacketListener; +import org.pcap4j.core.PcapPacket; +import org.pcap4j.packet.EthernetPacket; +import org.pcap4j.util.MacAddress; + +import java.util.*; + +/** + * Reassembles traffic flows at layer 2, i.e., for each combination of hosts, creates a list of packets exchanged + * between said hosts. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class Layer2FlowReassembler implements PacketListener { + + /** + * Maps a pair of MAC addresses to the packets exchanged between the two hosts. + * The key is the concatenation of the two MAC addresses in hex string format, where the lexicographically smaller + * MAC is at the front of the string. + */ + private final Map mFlows = new HashMap<>(); + + private final List mObservers = new ArrayList<>(); + + @Override + public void gotPacket(PcapPacket packet) { + // TODO: update to 802.11 packet...? + EthernetPacket ethPkt = packet.get(EthernetPacket.class); + + MacAddress srcAddr = ethPkt.getHeader().getSrcAddr(); + MacAddress dstAddr = ethPkt.getHeader().getDstAddr(); + + String key = keyFromAddresses(srcAddr, dstAddr); + // Create a new list if this pair of MAC addresses where not previously encountered and add packet to that list, + // or simply add to an existing list if one is present. + mFlows.computeIfAbsent(key, k -> { + Layer2Flow newFlow = new Layer2Flow(srcAddr, dstAddr); + // Inform observers of the new flow + mObservers.forEach(o -> o.onNewFlow(this, newFlow)); + return newFlow; + }).addPacket(packet); + } + + public void addObserver(Layer2FlowReassemblerObserver observer) { + mObservers.add(observer); + } + + public void removeObserver(Layer2FlowReassemblerObserver observer) { + mObservers.remove(observer); + } + + /** + * Get the traffic flow between two local endpoints ({@link MacAddress}es). + * @param addr1 The first endpoint. + * @param addr2 The second endpoint + * @return The traffic exchanged between the two endpoints. + */ + public Layer2Flow getFlowForAddresses(MacAddress addr1, MacAddress addr2) { + return mFlows.get(keyFromAddresses(addr1, addr2)); + } + + /** + * Get all traffic flows, i.e., a traffic flow for each unique pair of endpoints (MAC addresses). + * @return All traffic flows. + */ + public Collection getFlows() { + return mFlows.values(); + } + + /** + * Given two {@link MacAddress}es, generates the corresponding key string used in {@link #mFlows}. + * @param addr1 The first address. + * @param addr2 The second address. + * @return the key string used in {@link #mFlows} corresponding to the two addresses. + */ + private String keyFromAddresses(MacAddress addr1, MacAddress addr2) { + String addr1Str = addr1.toString(); + String addr2Str = addr2.toString(); + return addr1Str.compareTo(addr2Str) < 0 ? addr1Str + addr2Str : addr2Str + addr1Str; + } +} diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowReassemblerObserver.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowReassemblerObserver.java new file mode 100644 index 0000000..a71cd19 --- /dev/null +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer2/Layer2FlowReassemblerObserver.java @@ -0,0 +1,19 @@ +package edu.uci.iotproject.trafficreassembly.layer2; + +/** + * For observing a {@link Layer2FlowReassembler}. + * + * @author Janus Varmarken + */ +public interface Layer2FlowReassemblerObserver { + + /** + * Invoked when when a {@link Layer2FlowReassembler} detects a new flow (i.e., when it encounters traffic between two + * MAC addresses that has not previously communicated in the traffic trace). + * + * @param reassembler The {@link Layer2FlowReassembler} that detected the new flow. + * @param newFlow The new flow. + */ + void onNewFlow(Layer2FlowReassembler reassembler, Layer2Flow newFlow); + +} diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/Conversation.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/Conversation.java new file mode 100644 index 0000000..e89e81b --- /dev/null +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/Conversation.java @@ -0,0 +1,585 @@ +package edu.uci.iotproject.trafficreassembly.layer3; + +import edu.uci.iotproject.analysis.TcpConversationUtils; +import edu.uci.iotproject.util.PcapPacketUtils; +import org.pcap4j.core.PcapPacket; +import org.pcap4j.packet.IpV4Packet; +import org.pcap4j.packet.Packet; +import org.pcap4j.packet.TcpPacket; + +import java.util.*; + +/** + * Models a (TCP) conversation/connection/session/flow (packet's belonging to the same session between a client and a + * server). + * Holds a list of {@link PcapPacket}s identified as pertaining to the flow. Note that this list is not + * considered when determining equality of two {@code Conversation} instances in order to allow for a + * {@code Conversation} to function as a key in data structures such as {@link java.util.Map} and {@link java.util.Set}. + * See {@link #equals(Object)} for the definition of equality. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class Conversation { + + /* Begin instance properties */ + /** + * The IP of the host that is considered the client (i.e. the host that initiates the conversation) + * in this conversation. + */ + private final String mClientIp; + + /** + * The port number used by the host that is considered the client in this conversation. + */ + private final int mClientPort; + + /** + * The IP of the host that is considered the server (i.e. is the responder) in this conversation. + */ + private final String mServerIp; + + /** + * The port number used by the server in this conversation. + */ + private final int mServerPort; + + /** + * The list of packets (with payload) pertaining to this conversation. + */ + private final List mPackets; + + /** + * If {@link #isTls()} is {@code true}, this list contains the subset of {@link #mPackets} which are TLS Application + * Data packets. + */ + private final List mTlsApplicationDataPackets; + + /** + * Contains the sequence numbers used thus far by the host that is considered the client in this + * {@code Conversation}. + * Used for filtering out retransmissions. + */ + private final Set mSeqNumbersClient; + + /** + * Contains the sequence numbers used thus far by the host that is considered the server in this + * {@code Conversation}. + * Used for filtering out retransmissions. + */ + private final Set mSeqNumbersSrv; + + /** + * List of SYN packets pertaining to this conversation. + */ + private final List mSynPackets; + + /** + * List of pairs FINs and their corresponding ACKs associated with this conversation. + */ + private final List mFinPackets; + + /** + * List of RST packets associated with this conversation. + */ + private final List mRstPackets; + + /** + * Boolean to mark the packet as Application Data based on the previous packet that reaches MTU + */ + private boolean mApplicationData; + /* End instance properties */ + + /** + * Factory method for creating a {@code Conversation} from a {@link PcapPacket}. + * @param pcapPacket The {@code PcapPacket} that wraps a TCP segment for which a {@code Conversation} is to be initiated. + * @param clientIsSrc If {@code true}, the source address and source port found in the IP datagram and TCP segment + * wrapped in the {@code PcapPacket} are regarded as pertaining to the client, and the destination + * address and destination port are regarded as pertaining to the server---and vice versa if set + * to {@code false}. + * @return A {@code Conversation} initiated with ip:port for client and server according to the direction of the packet. + */ + public static Conversation fromPcapPacket(PcapPacket pcapPacket, boolean clientIsSrc) { + IpV4Packet ipPacket = pcapPacket.get(IpV4Packet.class); + TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class); + String clientIp = clientIsSrc ? ipPacket.getHeader().getSrcAddr().getHostAddress() : + ipPacket.getHeader().getDstAddr().getHostAddress(); + String srvIp = clientIsSrc ? ipPacket.getHeader().getDstAddr().getHostAddress() : + ipPacket.getHeader().getSrcAddr().getHostAddress(); + int clientPort = clientIsSrc ? tcpPacket.getHeader().getSrcPort().valueAsInt() : + tcpPacket.getHeader().getDstPort().valueAsInt(); + int srvPort = clientIsSrc ? tcpPacket.getHeader().getDstPort().valueAsInt() : + tcpPacket.getHeader().getSrcPort().valueAsInt(); + return new Conversation(clientIp, clientPort, srvIp, srvPort); + } + + /** + * Constructs a new {@code Conversation}. + * @param clientIp The IP of the host that is considered the client (i.e. the host that initiates the conversation) + * in the conversation. + * @param clientPort The port number used by the client for the conversation. + * @param serverIp The IP of the host that is considered the server (i.e. is the responder) in the conversation. + * @param serverPort The port number used by the server for the conversation. + */ + public Conversation(String clientIp, int clientPort, String serverIp, int serverPort) { + this.mClientIp = clientIp; + this.mClientPort = clientPort; + this.mServerIp = serverIp; + this.mServerPort = serverPort; + this.mPackets = new ArrayList<>(); + this.mTlsApplicationDataPackets = new ArrayList<>(); + this.mSeqNumbersClient = new HashSet<>(); + this.mSeqNumbersSrv = new HashSet<>(); + this.mSynPackets = new ArrayList<>(); + this.mFinPackets = new ArrayList<>(); + this.mRstPackets = new ArrayList<>(); + this.mApplicationData = false; + } + + /** + * Add a packet to the list of packets associated with this conversation. + * @param packet The packet that is to be added to (associated with) this conversation. + * @param ignoreRetransmissions Boolean value indicating if retransmissions should be ignored. + * If set to {@code true}, {@code packet} will not be added to the + * internal list of packets pertaining to this {@code Conversation} + * iff the sequence number of {@code packet} was already + * seen in a previous packet. + */ + public void addPacket(PcapPacket packet, boolean ignoreRetransmissions) { + // Precondition: verify that packet does indeed pertain to conversation. + onAddPrecondition(packet); + if (ignoreRetransmissions && isRetransmission(packet)) { + // Packet is a retransmission. Ignore it. + return; + } + // Select direction-dependent set of sequence numbers seen so far and update it with sequence number of new packet. + addSeqNumber(packet); + // Finally add packet to list of packets pertaining to this conversation. + mPackets.add(packet); + // Preserve order of packets in list: sort according to timestamp. + if (mPackets.size() > 1 && + mPackets.get(mPackets.size()-1).getTimestamp().isBefore(mPackets.get(mPackets.size()-2).getTimestamp())) { + Collections.sort(mPackets, (o1, o2) -> { + if (o1.getTimestamp().isBefore(o2.getTimestamp())) { return -1; } + else if (o2.getTimestamp().isBefore(o1.getTimestamp())) { return 1; } + else { return 0; } + }); + } + // If TLS, inspect packet to see if it's a TLS Application Data packet, and if so add it to the list of TLS + // Application Data packets. + if (isTls()) { + TcpPacket tcpPacket = packet.get(TcpPacket.class); + Packet tcpPayload = tcpPacket.getPayload(); + if (tcpPayload == null) { + return; + } + byte[] rawPayload = tcpPayload.getRawData(); + // The SSL record header is at the front of the payload and is 5 bytes long. + // The SSL record header type field (the first byte) is set to 23 if it is an Application Data packet. + if (rawPayload != null && rawPayload.length >= 5) { + if (rawPayload[0] == 23) { + mTlsApplicationDataPackets.add(packet); + // Consider the following packet a data packet if this packet's size == MTU size 1448 + if (rawPayload.length >= 1448) + mApplicationData = true; + } else if (rawPayload[0] == 20) { + // Do nothing for now - CHANGE_CIPHER_SPEC + } else if (rawPayload[0] == 21) { + // Do nothing for now - ALERT + } else if (rawPayload[0] == 22) { + // Do nothing for now - HANDSHAKE + } else { + // If it is TLS with payload, but rawPayload[0] != 23 + if (mApplicationData == true) { + // It is a continuation of the previous packet if the previous packet reaches MTU size 1448 and + // it is not either type 20, 21, or 22 + mTlsApplicationDataPackets.add(packet); + if (rawPayload.length < 1448) + mApplicationData = false; + } + } + } + } + } + + /** + * Get a list of packets pertaining to this {@code Conversation}. + * The returned list is a read-only list. + * @return the list of packets pertaining to this {@code Conversation}. + */ + public List getPackets() { + // Return read-only view to prevent external code from manipulating internal state (preserve invariant). + return Collections.unmodifiableList(mPackets); + } + + /** + * Records a TCP SYN packet as pertaining to this conversation (adds it to the the internal list). + * Attempts to add duplicate SYN packets will be ignored, and the caller is made aware of the attempt to add a + * duplicate by the return value being {@code false}. + * + * @param synPacket A {@link PcapPacket} wrapping a TCP SYN packet. + * @return {@code true} if the packet was successfully added to this {@code Conversation}, {@code false} otherwise. + */ + public boolean addSynPacket(PcapPacket synPacket) { + onAddPrecondition(synPacket); + final IpV4Packet synPacketIpSection = synPacket.get(IpV4Packet.class); + final TcpPacket synPacketTcpSection = synPacket.get(TcpPacket.class); + if (synPacketTcpSection == null || !synPacketTcpSection.getHeader().getSyn()) { + throw new IllegalArgumentException("Not a SYN packet."); + } + // We are only interested in recording one copy of the two SYN packets (one SYN packet in each direction), i.e., + // we want to discard retransmitted SYN packets. + if (mSynPackets.size() >= 2) { + return false; + } + // Check the set of recorded SYN packets to see if we have already recorded a SYN packet going in the same + // direction as the packet given in the argument. + boolean matchingPrevSyn = mSynPackets.stream().anyMatch(p -> { + IpV4Packet pIp = p.get(IpV4Packet.class); + TcpPacket pTcp = p.get(TcpPacket.class); + boolean srcAddrMatch = synPacketIpSection.getHeader().getSrcAddr().getHostAddress(). + equals(pIp.getHeader().getSrcAddr().getHostAddress()); + boolean dstAddrMatch = synPacketIpSection.getHeader().getDstAddr().getHostAddress(). + equals(pIp.getHeader().getDstAddr().getHostAddress()); + boolean srcPortMatch = synPacketTcpSection.getHeader().getSrcPort().valueAsInt() == + pTcp.getHeader().getSrcPort().valueAsInt(); + boolean dstPortMatch = synPacketTcpSection.getHeader().getDstPort().valueAsInt() == + pTcp.getHeader().getDstPort().valueAsInt(); + return srcAddrMatch && dstAddrMatch && srcPortMatch && dstPortMatch; + }); + if (matchingPrevSyn) { + return false; + } + // Update direction-dependent set of sequence numbers and record/log packet. + addSeqNumber(synPacket); + return mSynPackets.add(synPacket); + + /* + mSynPackets.stream().anyMatch(p -> { + IpV4Packet pIp = p.get(IpV4Packet.class); + TcpPacket pTcp = p.get(TcpPacket.class); + boolean srcAddrMatch = synPacketIpSection.getHeader().getSrcAddr().getHostAddress(). + equals(pIp.getHeader().getSrcAddr().getHostAddress()); + boolean dstAddrMatch = synPacketIpSection.getHeader().getDstAddr().getHostAddress(). + equals(pIp.getHeader().getDstAddr().getHostAddress()); + boolean srcPortMatch = synPacketTcpSection.getHeader().getSrcPort().valueAsInt() == + pTcp.getHeader().getSrcPort().valueAsInt(); + boolean dstPortMatch = synPacketTcpSection.getHeader().getDstPort().value() == + pTcp.getHeader().getDstPort().value(); + + boolean fourTupleMatch = srcAddrMatch && dstAddrMatch && srcPortMatch && dstPortMatch; + + boolean seqNoMatch = synPacketTcpSection.getHeader().getSequenceNumber() == + pTcp.getHeader().getSequenceNumber(); + + if (fourTupleMatch && !seqNoMatch) { + // If the four tuple that identifies the conversation matches, but the sequence number is different, + // it means that this SYN packet is, in fact, an attempt to establish a **new** connection, and hence + // the given packet is NOT part of this conversation, even though the ip:port combinations are (by + // chance) selected such that they match this conversation. + throw new IllegalArgumentException("Attempt to add SYN packet that belongs to a different conversation " + + "(which is identified by the same four tuple as this conversation)"); + } + return fourTupleMatch && seqNoMatch; + }); + */ + } + + /** + * Get a list of SYN packets pertaining to this {@code Conversation}. + * The returned list is a read-only list. + * @return the list of SYN packets pertaining to this {@code Conversation}. + */ + public List getSynPackets() { + return Collections.unmodifiableList(mSynPackets); + } + + /** + * Adds a TCP FIN packet to the list of TCP FIN packets associated with this conversation. + * @param finPacket The TCP FIN packet that is to be added to (associated with) this conversation. + */ + public void addFinPacket(PcapPacket finPacket) { + // Precondition: verify that packet does indeed pertain to conversation. + onAddPrecondition(finPacket); + // TODO: should call addSeqNumber here? + addSeqNumber(finPacket); + mFinPackets.add(new FinAckPair(finPacket)); + } + + /** + * Attempt to ACK any FIN packets held by this conversation. + * @param ackPacket The ACK for a FIN previously added to this conversation. + */ + public void attemptAcknowledgementOfFin(PcapPacket ackPacket) { + // Precondition: verify that the packet pertains to this conversation. + onAddPrecondition(ackPacket); + // Mark unack'ed FIN(s) that this ACK matches as ACK'ed (there might be more than one in case of retransmissions..?) + mFinPackets.replaceAll(finAckPair -> !finAckPair.isAcknowledged() && finAckPair.isCorrespondingAckPacket(ackPacket) ? new FinAckPair(finAckPair.getFinPacket(), ackPacket) : finAckPair); + } + + /** + * Retrieves an unmodifiable view of the list of {@link FinAckPair}s associated with this {@code Conversation}. + * @return an unmodifiable view of the list of {@link FinAckPair}s associated with this {@code Conversation}. + */ + public List getFinAckPairs() { + return Collections.unmodifiableList(mFinPackets); + } + + /** + * Get if this {@code Conversation} is considered to have been gracefully shut down. + * A {@code Conversation} has been gracefully shut down if it contains a FIN+ACK pair for both directions + * (client to server, and server to client). + * @return {@code true} if the connection has been gracefully shut down, false otherwise. + */ + public boolean isGracefullyShutdown() { + // The conversation has been gracefully shut down if we have recorded a FIN from both the client and the server which have both been ack'ed. + return mFinPackets.stream().anyMatch(finAckPair -> finAckPair.isAcknowledged() && PcapPacketUtils.isSource(finAckPair.getFinPacket(), mClientIp, mClientPort)) && + mFinPackets.stream().anyMatch(finAckPair -> finAckPair.isAcknowledged() && PcapPacketUtils.isSource(finAckPair.getFinPacket(), mServerIp, mServerPort)); + } + + /** + * Add a TCP segment for which the RST flag is set to this {@code Conversation}. + * @param packet A {@link PcapPacket} wrapping a TCP segment pertaining to this {@code Conversation} for which the + * RST flag is set. + */ + public void addRstPacket(PcapPacket packet) { + /* + * TODO: + * When now also keeping track of RST packets, should we also...? + * 1) Prevent later packets from being added once a RST segment has been added? + * 2) Extend 'isGracefullyShutdown()' to also consider RST segments, or add another method, 'isShutdown()' that + * both considers FIN/ACK (graceful) as well as RST (abrupt/"ungraceful") shutdown? + * 3) Should it be impossible to associate more than one RST segment with each Conversation? + */ + onAddPrecondition(packet); + TcpPacket tcpPacket = packet.get(TcpPacket.class); + if (tcpPacket == null || !tcpPacket.getHeader().getRst()) { + throw new IllegalArgumentException("not a RST packet"); + } + mRstPackets.add(packet); + } + + /** + * Get the TCP segments pertaining to this {@code Conversation} for which it was detected that the RST flag is set. + * @return the TCP segments pertaining to this {@code Conversation} for which it was detected that the RST flag is + * set. + */ + public List getRstPackets() { + return Collections.unmodifiableList(mRstPackets); + } + + // ========================================================================================================= + // We simply reuse equals and hashCode methods of String.class to be able to use this class as a key + // in a Map. + + /** + * Note: currently, equality is determined based on pairwise equality of the elements of the four tuple + * ({@link #mClientIp}, {@link #mClientPort}, {@link #mServerIp}, {@link #mServerPort}) for {@code this} and + * {@code obj}. + * @param obj The object to test for equality with {@code this}. + * @return {@code true} if {@code obj} is considered equal to {@code this} based on the definition of equality given above. + */ + @Override + public boolean equals(Object obj) { + return obj instanceof Conversation && this.toString().equals(obj.toString()); + } + + @Override + public int hashCode() { + return toString().hashCode(); + } + // ========================================================================================================= + + @Override + public String toString() { + return String.format("%s:%d %s:%d", mClientIp, mClientPort, mServerIp, mServerPort); + } + + /** + * Invoke to verify that the precondition holds when a caller attempts to add a packet to this {@code Conversation}. + * An {@link IllegalArgumentException} is thrown if the precondition is violated. + * @param packet the packet to be added to this {@code Conversation} + */ + private void onAddPrecondition(PcapPacket packet) { + // Apply precondition to preserve class invariant: all packets in mPackets must match the 4 tuple that + // defines the conversation. + IpV4Packet ipPacket = Objects.requireNonNull(packet.get(IpV4Packet.class)); + // For now we only support TCP flows. + TcpPacket tcpPacket = Objects.requireNonNull(packet.get(TcpPacket.class)); + String ipSrc = ipPacket.getHeader().getSrcAddr().getHostAddress(); + String ipDst = ipPacket.getHeader().getDstAddr().getHostAddress(); + int srcPort = tcpPacket.getHeader().getSrcPort().valueAsInt(); + int dstPort = tcpPacket.getHeader().getDstPort().valueAsInt(); + String clientIp, serverIp; + int clientPort, serverPort; + if (ipSrc.equals(mClientIp)) { + clientIp = ipSrc; + clientPort = srcPort; + serverIp = ipDst; + serverPort = dstPort; + } else { + clientIp = ipDst; + clientPort = dstPort; + serverIp = ipSrc; + serverPort = srcPort; + } + if (!(clientIp.equals(mClientIp) && clientPort == mClientPort && + serverIp.equals(mServerIp) && serverPort == mServerPort)) { + throw new IllegalArgumentException( + String.format("Attempt to add packet that does not pertain to %s", + Conversation.class.getSimpleName())); + } + } + + /** + *

+ * Determines if the TCP packet contained in {@code packet} is a retransmission of a previously seen (logged) + * packet. + *

+ * + * + * TODO: + * the current implementation, which uses a set of previously seen sequence numbers, will consider a segment + * with a reused sequence number---occurring as a result of sequence number wrap around for a very long-lived + * connection---as a retransmission (and may therefore end up discarding it even though it is in fact NOT a + * retransmission). Ideas? + * + * + * @param packet The packet. + * @return {@code true} if {@code packet} was determined to be a retransmission, {@code false} otherwise. + */ + public boolean isRetransmission(PcapPacket packet) { + // Extract sequence number. + int seqNo = packet.get(TcpPacket.class).getHeader().getSequenceNumber(); + switch (getDirection(packet)) { + case CLIENT_TO_SERVER: + return mSeqNumbersClient.contains(seqNo); + case SERVER_TO_CLIENT: + return mSeqNumbersSrv.contains(seqNo); + default: + throw new AssertionError(String.format("Unexpected value of enum '%s'", + Direction.class.getSimpleName())); + } + } + + /** + *

+ * Is this {@code Conversation} a TLS session? + *

+ * + * Note: the current implementation simply examines the port number(s) for 443; it does not verify if the + * application data is indeed encrypted. + * + * @return {@code true} if this {@code Conversation} is interpreted as a TLS session, {@code false} otherwise. + */ + public boolean isTls() { + /* + * TODO: + * - may want to change this to be "return mServerPort == 443 || mClientPort == 443;" in order to also detect + * TLS in those cases where it is not possible to correctly label who is the client and who is the server, + * i.e., when the trace does not contain the SYN/SYNACK exchange. + * - current implementation relies on the server using the conventional TLS port number; may instead want to + * inspect the first 4 bytes of each potential TLS packet to see if they match the SSL record header. + * + * 08/31/18: Added unconvetional TLS ports used by WeMo plugs and LiFX bulb. + * 09/20/18: Moved hardcoded ports to other class to allow other classes to query the set of TLS ports. + */ + return TcpConversationUtils.isTlsPort(mServerPort); + } + + /** + * If this {@code Conversation} is backing a TLS session (i.e., if the value of {@link #isTls()} is {@code true}), + * get the packets labeled as TLS Application Data packets. This is a subset of the full set of payload-carrying + * packets (as returned by {@link #getPackets()}). An exception is thrown if this method is invoked on a + * {@code Conversation} for which {@link #isTls()} returns {@code false}. + * + * @return A list containing exactly those packets that could be identified as TLS Application Data packets (through + * inspecting of the SSL record header). The list may be empty, if no TLS application data packets have been + * recorded for this {@code Conversation}. + */ + public List getTlsApplicationDataPackets() { + if (!isTls()) { + throw new NoSuchElementException("cannot get TLS Application Data packets for non-TLS TCP conversation"); + } + return Collections.unmodifiableList(mTlsApplicationDataPackets); + } + + /** + * Extracts the TCP sequence number from {@code packet} and adds it to the proper set of sequence numbers by + * analyzing the direction of the packet. + * @param packet A TCP packet (wrapped in a {@code PcapPacket}) that was added to this conversation and whose + * sequence number is to be recorded as seen. + */ + private void addSeqNumber(PcapPacket packet) { + // Note: below check is redundant if client code is correct as the call to check the precondition should already + // have been made by the addXPacket method that invokes this method. As such, the call below may be removed in + // favor of speed, but the improvement will be minor, hence the added safety may be worth it. + onAddPrecondition(packet); + // Extract sequence number. + int seqNo = packet.get(TcpPacket.class).getHeader().getSequenceNumber(); + // Determine direction of packet and add packet's sequence number to corresponding set of sequence numbers. + switch (getDirection(packet)) { + case CLIENT_TO_SERVER: + // Client to server packet. + mSeqNumbersClient.add(seqNo); + break; + case SERVER_TO_CLIENT: + // Server to client packet. + mSeqNumbersSrv.add(seqNo); + break; + default: + throw new AssertionError(String.format("Unexpected value of enum '%s'", + Direction.class.getSimpleName())); + } + } + + /** + * Determine the direction of {@code packet}. An {@link IllegalArgumentException} is thrown if {@code packet} does + * not pertain to this conversation. + * + * @param packet The packet whose direction is to be determined. + * @return A {@link Direction} indicating the direction of the packet. + */ + public Direction getDirection(PcapPacket packet) { + IpV4Packet ipPacket = packet.get(IpV4Packet.class); + String ipSrc = ipPacket.getHeader().getSrcAddr().getHostAddress(); + String ipDst = ipPacket.getHeader().getDstAddr().getHostAddress(); + // Determine direction of packet. + if (ipSrc.equals(mClientIp) && ipDst.equals(mServerIp)) { + // Client to server packet. + return Direction.CLIENT_TO_SERVER; + } else if (ipSrc.equals(mServerIp) && ipDst.equals(mClientIp)) { + // Server to client packet. + return Direction.SERVER_TO_CLIENT; + } else { + throw new IllegalArgumentException("getDirection: packet not related to " + getClass().getSimpleName()); + } + } + + /** + * Utility enum for expressing the direction of a packet pertaining to this {@code Conversation}. + */ + public enum Direction { + + CLIENT_TO_SERVER { + @Override + public String toCompactString() { + return "*"; + } + }, + SERVER_TO_CLIENT { + @Override + public String toCompactString() { + return ""; + } + }; + + /** + * Get a compact string representation of this {@code Direction}. + * @return a compact string representation of this {@code Direction}. + */ + abstract public String toCompactString(); + + } + +} diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/FinAckPair.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/FinAckPair.java new file mode 100644 index 0000000..fe4f032 --- /dev/null +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/FinAckPair.java @@ -0,0 +1,134 @@ +package edu.uci.iotproject.trafficreassembly.layer3; + +import org.pcap4j.core.PcapPacket; +import org.pcap4j.packet.IpV4Packet; +import org.pcap4j.packet.TcpPacket; + +/** + * Groups a FIN packet and its corresponding ACK packet. Immutable and thread safe. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class FinAckPair { + + private final PcapPacket mFinPacket; + private final PcapPacket mCorrespondingAckPacket; + + /** + * Constructs a {@code FinAckPair} given a FIN packet. + * The corresponding ACK packet field is set to {@code null}. + * @param finPacket A FIN packet. + */ + public FinAckPair(PcapPacket finPacket) { + if (!finPacket.get(TcpPacket.class).getHeader().getFin()) { + throw new IllegalArgumentException("not a FIN packet"); + } + mFinPacket = finPacket; + mCorrespondingAckPacket = null; + } + + /** + * Constructs a {@code FinAckPair} given a FIN and an ACK packet. + * @param finPacket A FIN packet. + * @param correspondingAckPacket The ACK packet corresponding to {@code finPacket}. + */ + public FinAckPair(PcapPacket finPacket, PcapPacket correspondingAckPacket) { + // Enforce class invariant, i.e. that the FIN and ACK are related. + // Note that it is indirectly checked whether finPacket is indeed a FIN packet + // as isCorrespondingAckPacket calls the single parameter constructor. + if (!FinAckPair.isCorrespondingAckPacket(finPacket, correspondingAckPacket)) { + throw new IllegalArgumentException("FIN and ACK not related"); + } + mFinPacket = finPacket; + mCorrespondingAckPacket = correspondingAckPacket; + } + + /** + * Get the FIN packet of this pair. + * @return the FIN packet of this pair. + */ + public PcapPacket getFinPacket() { + return mFinPacket; + } + + /** + * Get the corresponding ACK packet of this pair, if any. + * @return the corresponding ACK packet of this pair, if any. + */ + public PcapPacket getCorrespondingAckPacket() { + return mCorrespondingAckPacket; + } + + /** + * Was the FIN in this {@code FinAckPair} acknowledged? + * + * @return {@code true} if the corresponding ACK has been set in this {@code FinAckPair}. + */ + public boolean isAcknowledged() { + return mFinPacket != null && mCorrespondingAckPacket != null; + } + + /** + * Checks if a given packet is an ACK corresponding to the FIN packet in this {@code FinAckPair}. + * @return {@code true} if {@code packet} is an ACK that corresponds to the FIN in this pair, {@code false} otherwise. + */ + public boolean isCorrespondingAckPacket(PcapPacket packet) { + IpV4Packet inputIpPacket = packet.get(IpV4Packet.class); + TcpPacket inputTcpPacket = packet.get(TcpPacket.class); + if (inputIpPacket == null || inputTcpPacket == null || !inputTcpPacket.getHeader().getAck()) { + return false; + } + + IpV4Packet finIpPacket = mFinPacket.get(IpV4Packet.class); + TcpPacket finTcpPacket = mFinPacket.get(TcpPacket.class); + + // Extract (srcIp:port,dstIp:port) for input and member (FIN) packets. + String inputPacketIpSrc = inputIpPacket.getHeader().getSrcAddr().getHostAddress(); + String inputPacketIpDst = inputIpPacket.getHeader().getDstAddr().getHostAddress(); + int inputPacketPortSrc = inputTcpPacket.getHeader().getSrcPort().valueAsInt(); + int inputPacketPortDst = inputTcpPacket.getHeader().getDstPort().valueAsInt(); + String finPacketIpSrc = finIpPacket.getHeader().getSrcAddr().getHostAddress(); + String finPacketIpDst = finIpPacket.getHeader().getDstAddr().getHostAddress(); + int finPacketPortSrc = finTcpPacket.getHeader().getSrcPort().valueAsInt(); + int finPacketPortDst = finTcpPacket.getHeader().getDstPort().valueAsInt(); + + // For the two packets to be related, the dst of one must be the src of the other. + // Split into multiple if statements for readability. First check IP fields, then ports. + if (!(inputPacketIpDst.equals(finPacketIpSrc) && finPacketIpDst.equals(inputPacketIpSrc))) { + return false; + } + if (!(inputPacketPortDst == finPacketPortSrc && finPacketPortDst == inputPacketPortSrc)) { + return false; + } + + // Packets are (most likely) related (part of same conversation/stream). + // Now all that is left for us to check is if the sequence numbers match. + // Note: recall that the FIN packet advances the seq numbers by 1, + // so the ACK number will be one larger than the seq. number in the FIN packet. + return inputTcpPacket.getHeader().getAcknowledgmentNumber() == finTcpPacket.getHeader().getSequenceNumber() + 1; + } + + /** + * Static method to check if two given packets are a FIN and the corresponding ACK packet. + * The purpose of this method is a workaround to enforce the class invariant in the two parameter constructor. + * Specifically, the following should be avoided: + * 
+     *     public FinAckPair(PcapPacket finPacket, PcapPacket correspondingAckPacket) {
+     *         mFinPacket = finPacket;
+     *         // Below line is considered bad practice as the object has not been fully initialized at this stage.
+     *         if (!this.isCorrespondingAckPacket(correspondingAckPacket)) {
+     *             // ... throw exception
+     *         }
+     *     }
+     *
+ * @param finPacket The FIN packet. + * @param ackPacket The ACK packet that is to be checked if it corresponds to the given FIN packet. + * @return {@code true} if the ACK corresponds to the FIN, {@code false} otherwise. + */ + private static boolean isCorrespondingAckPacket(PcapPacket finPacket, PcapPacket ackPacket) { + FinAckPair tmp = new FinAckPair(finPacket); + return tmp.isCorrespondingAckPacket(ackPacket); + } + +} diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/TcpReassembler.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/TcpReassembler.java new file mode 100644 index 0000000..e150875 --- /dev/null +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/trafficreassembly/layer3/TcpReassembler.java @@ -0,0 +1,260 @@ +package edu.uci.iotproject.trafficreassembly.layer3; + +import org.pcap4j.core.PacketListener; +import org.pcap4j.core.PcapPacket; +import org.pcap4j.packet.*; + +import java.util.*; + +/** + * Reassembles TCP conversations (streams). + * Note: current version only supports TCP over IPv4. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class TcpReassembler implements PacketListener { + + /** + * Holds open {@link Conversation}s, i.e., {@code Conversation}s that have not been detected as + * (gracefully) terminated based on the set of packets observed thus far. + * A {@link Conversation} is moved to {@link #mTerminatedConversations} if it can be determined that it is has + * terminated. Termination can be detected by a) observing two {@link FinAckPair}s, one in each direction, (graceful + * termination, see {@link Conversation#isGracefullyShutdown()}) or b) by observing a SYN packet that matches the + * four tuple of an existing {@code Conversation}, but which holds a different sequence number than the + * same-direction SYN packet recorded for the {@code Conversation}. + *

+ * Note that due to limitations of the {@link Set} interface (specifically, there is no {@code get(T t)} method), + * we have to resort to a {@link Map} (in which keys map to themselves) to "mimic" a set with {@code get(T t)} + * functionality. + * + * @see this question on StackOverflow.com + */ + private final Map mOpenConversations = new HashMap<>(); + + /** + * Holds terminated {@link Conversation}s. + */ + private final List mTerminatedConversations = new ArrayList<>(); + + @Override + public void gotPacket(PcapPacket pcapPacket) { + IpV4Packet ipPacket = pcapPacket.get(IpV4Packet.class); + TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class); + + if (ipPacket == null || tcpPacket == null) { + return; + } + // ... TODO? + processPacket(pcapPacket); +// Class clazz = pcapPacket.getClass(); +// RadiotapPacket radiotapPacket = pcapPacket.get(RadiotapPacket.class); +// Dot11ManagementPacket dot11ManagementPacket = pcapPacket.get(Dot11ManagementPacket.class); +// if (dot11ManagementPacket != null) { +// return; +// } +// if (radiotapPacket != null) { +// processRadiotapPacket(pcapPacket); +// } + } + + /** + * Get the reassembled TCP connections. Note that if this is called while packets are still being processed (by + * calls to {@link #gotPacket(PcapPacket)}), the behavior is undefined and the returned list may be inconsistent. + * @return The reassembled TCP connections. + */ + public List getTcpConversations() { + ArrayList combined = new ArrayList<>(); + combined.addAll(mTerminatedConversations); + combined.addAll(mOpenConversations.values()); + return combined; + } + + private void processRadiotapPacket(PcapPacket pcapPacket) { + RadiotapPacket radiotapPacket = pcapPacket.get(RadiotapPacket.class); + + RadiotapPacket.RadiotapHeader header = radiotapPacket.getHeader(); + short length = header.getLength(); + ArrayList radiotapData = header.getDataFields(); + // TODO: We can handle this 802.11 QoS data by creating our own class + // TODO: We only need to handle the first few bytes for source, destination, receiver, and transmitter + // TODO: addresses + Packet dataPacket = radiotapPacket.getPayload(); + int dataLength = dataPacket.length(); + } + + private void processPacket(PcapPacket pcapPacket) { + TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class); + // Handle client connection initiation attempts. + if (tcpPacket.getHeader().getSyn() && !tcpPacket.getHeader().getAck()) { + // A segment with the SYN flag set, but no ACK flag indicates that a client is attempting to initiate a new + // connection. + processNewConnectionRequest(pcapPacket); + return; + } + // Handle server connection initiation acknowledgement + if (tcpPacket.getHeader().getSyn() && tcpPacket.getHeader().getAck()) { + // A segment with both the SYN and ACK flags set indicates that the server has accepted the client's request + // to initiate a new connection. + processNewConnectionAck(pcapPacket); + return; + } + // Handle resets + if (tcpPacket.getHeader().getRst()) { + processRstPacket(pcapPacket); + return; + } + // Handle FINs + if (tcpPacket.getHeader().getFin()) { + // Handle FIN packet. + processFinPacket(pcapPacket); + } + // Handle ACKs (currently only ACKs of FINS) + if (tcpPacket.getHeader().getAck()) { + processAck(pcapPacket); + } + // Handle packets that carry payload (application data). + if (tcpPacket.getPayload() != null) { + processPayloadPacket(pcapPacket); + } + } + + private void processNewConnectionRequest(PcapPacket clientSynPacket) { + // A SYN w/o ACK always originates from the client. + Conversation conv = Conversation.fromPcapPacket(clientSynPacket, true); + conv.addSynPacket(clientSynPacket); + // Is there an ongoing conversation for the same four tuple (clientIp, clientPort, serverIp, serverPort) as + // found in the new SYN packet? + Conversation ongoingConv = mOpenConversations.get(conv); + if (ongoingConv != null) { + if (ongoingConv.isRetransmission(clientSynPacket)) { + // SYN retransmission detected, do nothing. + return; + // TODO: the way retransmission detection is implemented may cause a bug for connections where we have + // not recorded the initial SYN, but only the SYN ACK, as retransmission is determined by comparing the + // sequence numbers of initial SYNs -- and if no initial SYN is present for the Conversation, the new + // SYN will be interpreted as a retransmission. Possible fix: let isRentransmission ALWAYS return false + // when presented with a SYN packet when the Conversation already holds a SYN ACK packet? + } else { + // New SYN has different sequence number than SYN recorded for ongoingConv, so this must be an attempt + // to establish a new conversation with the same four tuple as ongoingConv. + // Mark existing connection as terminated. + // TODO: is this 100% theoretically correct, e.g., if many connection attempts are made back to back? And RST packets? + mTerminatedConversations.add(ongoingConv); + mOpenConversations.remove(ongoingConv); + } + } + // Finally, update the map of open connections with the new connection. + mOpenConversations.put(conv, conv); + } + + + /* + * TODO a problem across the board for all processXPacket methods below: + * if we start the capture in the middle of a TCP connection, we will not have an entry for the conversation in the + * map as we have not seen the initial SYN packet. + * Two ways we can address this: + * a) Perform null-checks and ignore packets for which we have not seen SYN + * + easy to get correct + * - we discard data (issue for long-lived connections!) + * b) Add a corresponding conversation entry whenever we encounter a packet that does not map to a conversation + * + we consider all data + * - not immediately clear if this will introduce bugs (incorrectly mapping packets to wrong conversations?) + * + * [[[ I went with option b) for now; see getOngoingConversationOrCreateNew(PcapPacket pcapPacket). ]]] + */ + + private void processNewConnectionAck(PcapPacket srvSynPacket) { + // Find the corresponding ongoing connection, if any (if we start the capture just *after* the initial SYN, no + // ongoing conversation entry will exist, so it must be created in that case). +// Conversation conv = mOpenConversations.get(Conversation.fromPcapPacket(srvSynPacket, false)); + Conversation conv = getOngoingConversationOrCreateNew(srvSynPacket); + // Note: exploits &&'s short-circuit operation: only attempts to add non-retransmissions. + if (!conv.isRetransmission(srvSynPacket) && !conv.addSynPacket(srvSynPacket)) { + // For safety/debugging: if NOT a retransmission and add fails, + // something has gone terribly wrong/invariant is broken. +// throw new AssertionError("Attempt to add SYN ACK packet that was NOT a retransmission failed." + +// Conversation.class.getSimpleName() + " invariant broken."); + } + } + + private void processRstPacket(PcapPacket rstPacket) { + Conversation conv = getOngoingConversationOrCreateNew(rstPacket); + // Add RST packet to conversation. + conv.addRstPacket(rstPacket); + // Move conversation to set of terminated conversations. + mTerminatedConversations.add(conv); + mOpenConversations.remove(conv, conv); + } + + private void processFinPacket(PcapPacket finPacket) { +// getOngoingConversationForPacket(finPacket).addFinPacket(finPacket); + getOngoingConversationOrCreateNew(finPacket).addFinPacket(finPacket); + } + + private void processAck(PcapPacket ackPacket) { +// getOngoingConversationForPacket(ackPacket).attemptAcknowledgementOfFin(ackPacket); + // Note that unlike the style for SYN, FIN, and payload packets, for "ACK only" packets, we want to avoid + // creating a new conversation. + Conversation conv = getOngoingConversationForPacket(ackPacket); + if (conv != null) { + // The ACK may be an ACK of a FIN, so attempt to mark the FIN as ack'ed. + conv.attemptAcknowledgementOfFin(ackPacket); + if (conv.isGracefullyShutdown()) { + // Move conversation to set of terminated conversations. + mTerminatedConversations.add(conv); + mOpenConversations.remove(conv); + } + } + // Note: add (additional) processing of ACKs (that are not ACKs of FINs) as necessary here... + } + + private void processPayloadPacket(PcapPacket pcapPacket) { +// getOngoingConversationForPacket(pcapPacket).addPacket(pcapPacket, true); + getOngoingConversationOrCreateNew(pcapPacket).addPacket(pcapPacket, true); + } + + /** + * Locates an ongoing conversation (if any) that {@code pcapPacket} pertains to. + * @param pcapPacket The packet that is to be mapped to an ongoing {@code Conversation}. + * @return The {@code Conversation} matching {@code pcapPacket} or {@code null} if there is no match. + */ + private Conversation getOngoingConversationForPacket(PcapPacket pcapPacket) { + // We cannot know if this is a client-to-server or server-to-client packet without trying both options... + Conversation conv = mOpenConversations.get(Conversation.fromPcapPacket(pcapPacket, true)); + if (conv == null) { + conv = mOpenConversations.get(Conversation.fromPcapPacket(pcapPacket, false)); + } + return conv; + } + + /** + * Like {@link #getOngoingConversationForPacket(PcapPacket)}, but creates and inserts a new {@code Conversation} + * into {@link #mOpenConversations} if no open conversation is found (i.e., in the case that + * {@link #getOngoingConversationForPacket(PcapPacket)} returns {@code null}). + * + * @param pcapPacket The packet that is to be mapped to an ongoing {@code Conversation}. + * @return The existing, ongoing {@code Conversation} matching {@code pcapPacket} or the newly created one in case + * no match was found. + */ + private Conversation getOngoingConversationOrCreateNew(PcapPacket pcapPacket) { + Conversation conv = getOngoingConversationForPacket(pcapPacket); + if (conv == null) { + TcpPacket tcpPacket = pcapPacket.get(TcpPacket.class); + if (tcpPacket.getHeader().getSyn() && tcpPacket.getHeader().getAck()) { + // A SYN ACK packet always originates from the server (it is a reply to the initial SYN packet from the client) + conv = Conversation.fromPcapPacket(pcapPacket, false); + } else { + // TODO: can we do anything else but arbitrarily select who is designated as the server in this case? + // We can check if the IP prefix matches a local IP when handling traffic observed inside the local + // network, but that obviously won't be a useful strategy for an observer at the WAN port. + String srcIp = pcapPacket.get(IpV4Packet.class).getHeader().getSrcAddr().getHostAddress(); + // TODO: REPLACE THE ROUTER'S IP WITH A PARAMETER!!! + boolean clientIsSrc = srcIp.startsWith("10.") || srcIp.startsWith("192.168.") || srcIp.equals("128.195.205.105"); + conv = Conversation.fromPcapPacket(pcapPacket, clientIsSrc); + } + mOpenConversations.put(conv, conv); + } + return conv; + } +} diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java index 8f58d97..cee09fe 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java @@ -1,13 +1,15 @@ package edu.uci.iotproject.util; -import edu.uci.iotproject.Conversation; +import edu.uci.iotproject.trafficreassembly.layer3.Conversation; import edu.uci.iotproject.analysis.PcapPacketPair; import edu.uci.iotproject.analysis.TcpConversationUtils; import edu.uci.iotproject.analysis.TriggerTrafficExtractor; import org.apache.commons.math3.stat.clustering.Cluster; import org.pcap4j.core.PcapPacket; +import org.pcap4j.packet.EthernetPacket; import org.pcap4j.packet.IpV4Packet; import org.pcap4j.packet.TcpPacket; +import org.pcap4j.util.MacAddress; import java.util.*; @@ -26,6 +28,25 @@ public final class PcapPacketUtils { */ private static final int SIGNATURE_MERGE_THRESHOLD = 5; + + /** + * Gets the source address of the Ethernet part of {@code packet}. + * @param packet The packet for which the Ethernet source address is to be extracted. + * @return The source address of the Ethernet part of {@code packet}. + */ + public static MacAddress getEthSrcAddr(PcapPacket packet) { + return getEthernetPacketOrThrow(packet).getHeader().getSrcAddr(); + } + + /** + * Gets the destination address of the Ethernet part of {@code packet}. + * @param packet The packet for which the Ethernet destination address is to be extracted. + * @return The destination address of the Ethernet part of {@code packet}. + */ + public static MacAddress getEthDstAddr(PcapPacket packet) { + return getEthernetPacketOrThrow(packet).getHeader().getDstAddr(); + } + /** * Determines if a given {@link PcapPacket} wraps a {@link TcpPacket}. * @param packet The {@link PcapPacket} to inspect. @@ -346,7 +367,7 @@ public final class PcapPacketUtils { /** * Gets the {@link IpV4Packet} contained in {@code packet}, or throws a {@link NullPointerException} if * {@code packet} does not contain an {@link IpV4Packet}. - * @param packet A {@link PcapPacket} that is expected to contain a {@link IpV4Packet}. + * @param packet A {@link PcapPacket} that is expected to contain an {@link IpV4Packet}. * @return The {@link IpV4Packet} contained in {@code packet}. * @throws NullPointerException if {@code packet} does not encapsulate an {@link IpV4Packet}. */ @@ -354,6 +375,17 @@ public final class PcapPacketUtils { return Objects.requireNonNull(packet.get(IpV4Packet.class), "not an IPv4 packet"); } + /** + * Gets the {@link EthernetPacket} contained in {@code packet}, or throws a {@link NullPointerException} if + * {@code packet} does not contain an {@link EthernetPacket}. + * @param packet A {@link PcapPacket} that is expected to contain an {@link EthernetPacket}. + * @return The {@link EthernetPacket} contained in {@code packet}. + * @throws NullPointerException if {@code packet} does not encapsulate an {@link EthernetPacket}. + */ + private static final EthernetPacket getEthernetPacketOrThrow(PcapPacket packet) { + return Objects.requireNonNull(packet.get(EthernetPacket.class), "not an Ethernet packet"); + } + /** * Print signatures in {@code List} of {@code List} of {@code List} of {@code PcapPacket} objects. *