package edu.uci.iotproject.analysis;
-import edu.uci.iotproject.Conversation;
+import edu.uci.iotproject.trafficreassembly.layer3.Conversation;
import edu.uci.iotproject.DnsMap;
import edu.uci.iotproject.util.PcapPacketUtils;
import org.pcap4j.core.PcapPacket;
import org.pcap4j.packet.TcpPacket;
import java.util.*;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static edu.uci.iotproject.util.PcapPacketUtils.*;
/**
* Utility functions for analyzing and structuring (sets of) {@link Conversation}s.
*/
public class TcpConversationUtils {
+ /**
+ * Identifies the adjacency type of the signature for merging.
+ */
+ public enum SignaturePosition {
+ NOT_ADJACENT,
+ LEFT_ADJACENT,
+ RIGHT_ADJACENT
+ }
/**
* <p>
* Given a {@link Conversation}, extract its set of "packet pairs", i.e., pairs of request-reply packets.
+ * <em>The extracted pairs are formed from the full set of payload-carrying TCP packets.</em>
* </p>
*
* <b>Note:</b> in the current implementation, if one endpoint sends multiple packets back-to-back with no
* interleaved reply packets from the other endpoint, such packets are converted to one-item pairs (i.e., instances
- * of {@lin PcapPacketPair} where {@link PcapPacketPair#getSecond()} is {@code null}).
+ * of {@link PcapPacketPair} where {@link PcapPacketPair#getSecond()} is {@code null}).
*
* @param conv The {@code Conversation} for which packet pairs are to be extracted.
* @return The packet pairs extracted from {@code conv}.
*/
public static List<PcapPacketPair> extractPacketPairs(Conversation conv) {
- List<PcapPacket> packets = conv.getPackets();
+ return extractPacketPairs(conv.getPackets());
+ }
+
+
+ /**
+ * <p>
+ * Given a {@link Conversation}, extract its set of "packet pairs", i.e., pairs of request-reply packets.
+ * <em>The extracted pairs are formed from the full set of TLS Application Data packets.</em>
+ * </p>
+ *
+ * <b>Note:</b> in the current implementation, if one endpoint sends multiple packets back-to-back with no
+ * interleaved reply packets from the other endpoint, such packets are converted to one-item pairs (i.e., instances
+ * of {@link PcapPacketPair} where {@link PcapPacketPair#getSecond()} is {@code null}).
+ *
+ * @param conv The {@code Conversation} for which packet pairs are to be extracted.
+ * @return The packet pairs extracted from {@code conv}.
+ */
+ public static List<PcapPacketPair> extractTlsAppDataPacketPairs(Conversation conv) {
+ if (!conv.isTls()) {
+ throw new IllegalArgumentException(String.format("Provided %s argument is not a TLS session"));
+ }
+ return extractPacketPairs(conv.getTlsApplicationDataPackets());
+ }
+
+ // Helper method for implementing the public API of similarly named methods.
+ private static List<PcapPacketPair> extractPacketPairs(List<PcapPacket> packets) {
List<PcapPacketPair> pairs = new ArrayList<>();
+// for(PcapPacket pp : packets) {
+// System.out.print(pp.length() + " ");
+// }
+// System.out.println();
+
int i = 0;
while (i < packets.size()) {
PcapPacket p1 = packets.get(i);
pairs.add(new PcapPacketPair(p1, p2));
// Advance two packets as we have already processed the packet at index i+1 in order to create the pair.
i += 2;
+ //i++;
}
} else {
// Last packet of conversation => one item pair
return result;
}
+ /**
+ * Given a {@link Collection} of {@link Conversation}s, builds a {@link Map} from {@link String} to {@link List}
+ * of {@link Conversation}s such that each key is the <em>concatenation of the packet lengths of all payload packets
+ * (i.e., the set of packets returned by {@link Conversation#getPackets()}) separated by a delimiter</em> of any
+ * {@link Conversation} pointed to by that key. In other words, what the {@link Conversation}s {@code cs} pointed to
+ * by the key {@code s} have in common is that they all contain exactly the same number of payload packets <em>and
+ * </em> these payload packets are identical across all {@code Conversation}s in {@code cs} in terms of packet
+ * length and packet order. For example, if the key is "152 440 550", this means that every individual
+ * {@code Conversation} in the list of {@code Conversation}s pointed to by that key contain exactly three payload
+ * packet of lengths 152, 440, and 550, and these three packets are ordered in the order prescribed by the key.
+ *
+ * @param conversations The collection of {@code Conversation}s to group by packet sequence.
+ * @param verbose If set to {@code true}, the grouping (and therefore the key) will also include SYN/SYNACK,
+ * FIN/FINACK, RST packets, and each payload-carrying packet will have an indication of the direction
+ * of the packet prepended.
+ * @return a {@link Map} from {@link String} to {@link List} of {@link Conversation}s such that each key is the
+ * <em>concatenation of the packet lengths of all payload packets (i.e., the set of packets returned by
+ * {@link Conversation#getPackets()}) separated by a delimiter</em> of any {@link Conversation} pointed to
+ * by that key.
+ */
+ public static Map<String, List<Conversation>> groupConversationsByPacketSequence(Collection<Conversation> conversations, boolean verbose) {
+ return conversations.stream().collect(Collectors.groupingBy(c -> toSequenceString(c, verbose)));
+ }
+
+ public static Map<String, List<Conversation>> groupConversationsByTlsApplicationDataPacketSequence(Collection<Conversation> conversations) {
+ return conversations.stream().collect(Collectors.groupingBy(
+ c -> c.getTlsApplicationDataPackets().stream().map(p -> Integer.toString(p.getOriginalLength())).
+ reduce("", (s1, s2) -> s1.length() == 0 ? s2 : s1 + " " + s2))
+ );
+ }
+
/**
* Given a {@link Conversation}, counts the frequencies of each unique packet length seen as part of the
* {@code Conversation}.
}
return result;
}
+
+ /**
+ * Given a {@link Conversation}, extract its packet length sequence.
+ * @param c The {@link Conversation} from which a packet length sequence is to be extracted.
+ * @return An {@code Integer[]} that holds the packet lengths of all payload-carrying packets in {@code c}. The
+ * packet lengths in the returned array are ordered by packet timestamp.
+ */
+ public static Integer[] getPacketLengthSequence(Conversation c) {
+ return getPacketLengthSequence(c.getPackets());
+ }
+
+
+ /**
+ * Given a {@link Conversation}, extract its packet length sequence, but only include packet lengths of those
+ * packets that carry TLS Application Data.
+ * @param c The {@link Conversation} from which a TLS Application Data packet length sequence is to be extracted.
+ * @return An {@code Integer[]} that holds the packet lengths of all packets in {@code c} that carry TLS Application
+ * Data. The packet lengths in the returned array are ordered by packet timestamp.
+ */
+ public static Integer[] getPacketLengthSequenceTlsAppDataOnly(Conversation c) {
+ if (!c.isTls()) {
+ throw new IllegalArgumentException("Provided " + c.getClass().getSimpleName() + " was not a TLS session");
+ }
+ return getPacketLengthSequence(c.getTlsApplicationDataPackets());
+ }
+
+ /**
+ * Given a list of packets, extract the packet lengths and wrap them in an array such that the packet lengths in the
+ * resulting array appear in the same order as their corresponding packets in the input list.
+ * @param packets The list of packets for which the packet lengths are to be extracted.
+ * @return An array containing the packet lengths in the same order as their corresponding packets in the input list.
+ */
+ private static Integer[] getPacketLengthSequence(List<PcapPacket> packets) {
+ return packets.stream().map(pkt -> pkt.getOriginalLength()).toArray(Integer[]::new);
+ }
+
+ /**
+ * Builds a string representation of the sequence of packets exchanged as part of {@code c}.
+ * @param c The {@link Conversation} for which a string representation of the packet sequence is to be constructed.
+ * @param verbose {@code true} if set to true, the returned sequence string will also include SYN/SYNACK,
+ * FIN/FINACK, RST packets, as well as an indication of the direction of payload-carrying packets.
+ * @return a string representation of the sequence of packets exchanged as part of {@code c}.
+ */
+ private static String toSequenceString(Conversation c, boolean verbose) {
+ // Payload-parrying packets are always included, but only prepend direction if verbose output is chosen.
+ Stream<String> s = c.getPackets().stream().map(p -> verbose ? c.getDirection(p).toCompactString() + p.getOriginalLength() : Integer.toString(p.getOriginalLength()));
+ if (verbose) {
+ // In the verbose case, we also print SYN, FIN and RST packets.
+ // Convert the SYN packets to a string representation and prepend them in front of the payload packets.
+ s = Stream.concat(c.getSynPackets().stream().map(p -> isSyn(p) && isAck(p) ? "SYNACK" : "SYN"), s);
+ // Convert the FIN packets to a string representation and append them after the payload packets.
+ s = Stream.concat(s, c.getFinAckPairs().stream().map(f -> f.isAcknowledged() ? "FINACK" : "FIN"));
+ // Convert the RST packets to a string representation and append at the end.
+ s = Stream.concat(s, c.getRstPackets().stream().map(r -> "RST"));
+ }
+ /*
+ * Note: the collector internally uses a StringBuilder, which is more efficient than simply doing string
+ * concatenation as in the following example:
+ * s.reduce("", (s1, s2) -> s1.length() == 0 ? s2 : s1 + " " + s2);
+ * (above code is O(N^2) where N is the number of characters)
+ */
+ return s.collect(Collectors.joining(" "));
+ }
+
+ /**
+ * Set of port numbers that we consider TLS traffic.
+ * Note: purposefully initialized as a {@link HashSet} to get O(1) {@code contains()} call.
+ */
+ private static final Set<Integer> TLS_PORTS = Stream.of(443, 8443, 41143).
+ collect(Collectors.toCollection(HashSet::new));
+
+ /**
+ * Check if a given port number is considered a TLS port.
+ * @param port The port number to check.
+ * @return {@code true} if the port number is considered a TLS port, {@code false} otherwise.
+ */
+ public static boolean isTlsPort(int port) {
+ return TLS_PORTS.contains(port);
+ }
+
+ /**
+ * Appends a space to {@code sb} <em>iff</em> {@code sb} already contains some content.
+ * @param sb A {@link StringBuilder} that should have a space appended <em>iff</em> it is not empty.
+ */
+ private static void appendSpaceIfNotEmpty(StringBuilder sb) {
+ if (sb.length() != 0) {
+ sb.append(" ");
+ }
+ }
+
+ /**
+ * Given a list of {@link Conversation} objects, sort them by timestamps.
+ * @param conversations The list of {@link Conversation} objects to be sorted.
+ * @return A sorted list of {@code Conversation} based on timestamps of the first
+ * packet in the {@code Conversation}.
+ */
+ public static List<Conversation> sortConversationList(List<Conversation> conversations) {
+ // Get rid of Conversation objects with no packets.
+ conversations.removeIf(x -> x.getPackets().size() == 0);
+ // Sort the list based on the first packet's timestamp!
+ Collections.sort(conversations, (c1, c2) ->
+ c1.getPackets().get(0).getTimestamp().compareTo(c2.getPackets().get(0).getTimestamp()));
+ return conversations;
+ }
+
+ /**
+ * Given a {@code List} of {@link Conversation} objects, find one that has the given {@code List}
+ * of {@code PcapPacket}.
+ * @param conversations The {@code List} of {@link Conversation} objects as reference.
+ * @param ppList The {@code List} of {@code PcapPacket} objects to search in the {@code List} of {@link Conversation}.
+ * @return A {@code Conversation} that contains the given {@code List} of {@code PcapPacket}.
+ */
+ public static Conversation returnConversation(List<PcapPacket> ppList, List<Conversation> conversations) {
+ // TODO: This part of comparison takes into account that the list of conversations is not sorted
+ // TODO: We could optimize this to have a better performance by requiring a sorted-by-timestamp list
+ // TODO: as a parameter
+ // Find a Conversation that ppList is part of
+ for (Conversation c : conversations) {
+ // Figure out if c is the Conversation that ppList is in
+ if (isPartOfConversation(ppList, c)) {
+ return c;
+ }
+ }
+ // Return null if not found
+ return null;
+ }
+
+ /**
+ * Given a {@link Conversation} objects, check if {@code List} of {@code PcapPacket} is part of it and return the
+ * adjacency label based on {@code SignaturePosition}.
+ * @param conversation The {@link Conversation} object as reference.
+ * @param ppListFirst The first {@code List} of {@code PcapPacket} objects in the {@link Conversation}.
+ * @param ppListSecond The second {@code List} of {@code PcapPacket} objects in the {@link Conversation} whose
+ * position will be observed in the {@link Conversation} with respect to ppListFirst.
+ * @return A {@code SignaturePosition} that represents the position of the signature against another signature
+ * in a {@link Conversation}.
+ */
+ public static SignaturePosition isPartOfConversationAndAdjacent(List<PcapPacket> ppListFirst,
+ List<PcapPacket> ppListSecond,
+ Conversation conversation) {
+ // Take the first element in ppList and compare it
+ // The following elements in ppList are guaranteed to be in the same Conversation
+ // TODO: This part of comparison takes into account that the list of conversations is not sorted
+ // TODO: We could optimize this to have a better performance by requiring a sorted-by-timestamp list
+ // TODO: as a parameter
+ if (isPartOfConversation(ppListSecond, conversation)) {
+ // Compare the first element of ppListSecond with the last element of ppListFirst to know
+ // whether ppListSecond is RIGHT_ADJACENT relative to ppListFirst.
+ PcapPacket lastElOfFirstList = ppListFirst.get(ppListFirst.size() - 1);
+ PcapPacket firstElOfSecondList = ppListSecond.get(0);
+ // If the positions of the two are in order, then they are adjacent.
+ int indexOfLastElOfFirstList = returnIndexInConversation(lastElOfFirstList, conversation);
+ int indexOfFirstElOfSecondList = returnIndexInConversation(firstElOfSecondList, conversation);
+ if(indexOfLastElOfFirstList + 1 == indexOfFirstElOfSecondList) {
+ return SignaturePosition.RIGHT_ADJACENT;
+ }
+ // NOT RIGHT_ADJACENT, so check for LEFT_ADJACENT.
+ // Compare the first element of ppListRight with the last element of ppListSecond to know
+ // whether ppListSecond is LEFT_ADJACENT relative to ppListFirst.
+ PcapPacket firstElOfFirstList = ppListFirst.get(0);
+ PcapPacket lastElOfSecondList = ppListSecond.get(ppListSecond.size() - 1);
+ // If the positions of the two are in order, then they are adjacent.
+ int indexOfFirstElOfFirstList = returnIndexInConversation(firstElOfFirstList, conversation);
+ int indexOfLastElOfSecondList = returnIndexInConversation(lastElOfSecondList, conversation);
+ if(indexOfLastElOfSecondList + 1 == indexOfFirstElOfFirstList) {
+ return SignaturePosition.LEFT_ADJACENT;
+ }
+ }
+ // Return NOT_ADJACENT if not found.
+ return SignaturePosition.NOT_ADJACENT;
+ }
+
+ /**
+ * Given a {@link Conversation} objects, check if {@code List} of {@code PcapPacket} is part of it.
+ * @param conversation The {@link Conversation} object as reference.
+ * @param ppList The {@code List} of {@code PcapPacket} objects to search in the {@link Conversation}.
+ * @return A {@code Boolean} value that represents the presence of the {@code List} of {@code PcapPacket} in
+ * the {@link Conversation}.
+ */
+ private static boolean isPartOfConversation(List<PcapPacket> ppList, Conversation conversation) {
+ // Find the first element of ppList in conversation.
+ if (conversation.getPackets().contains(ppList.get(0)))
+ return true;
+ // Return false if not found.
+ return false;
+ }
+
+ /**
+ * Given a {@link Conversation} objects, check the index of a {@code PcapPacket} in it.
+ * @param conversation The {@link Conversation} object as reference.
+ * @param pp The {@code PcapPacket} object to search in the {@link Conversation}.
+ * @return An {@code Integer} value that gives the index of the {@code PcapPacket} in the {@link Conversation}.
+ */
+ private static int returnIndexInConversation(PcapPacket pp, Conversation conversation) {
+ // Find pp in conversation.
+ if (conversation.getPackets().contains(pp))
+ return conversation.getPackets().indexOf(pp);
+ // Return -1 if not found.
+ return -1;
+ }
}
projects / pingpong.git / blobdiff