1 package edu.uci.iotproject.analysis;
3 import edu.uci.iotproject.Conversation;
4 import edu.uci.iotproject.DnsMap;
5 import edu.uci.iotproject.FinAckPair;
6 import edu.uci.iotproject.util.PcapPacketUtils;
7 import org.pcap4j.core.PcapPacket;
8 import org.pcap4j.packet.IpV4Packet;
9 import org.pcap4j.packet.TcpPacket;
14 * Utility functions for analyzing and structuring (sets of) {@link Conversation}s.
16 * @author Janus Varmarken {@literal <jvarmark@uci.edu>}
17 * @author Rahmadi Trimananda {@literal <rtrimana@uci.edu>}
19 public class TcpConversationUtils {
23 * Given a {@link Conversation}, extract its set of "packet pairs", i.e., pairs of request-reply packets.
26 * <b>Note:</b> in the current implementation, if one endpoint sends multiple packets back-to-back with no
27 * interleaved reply packets from the other endpoint, such packets are converted to one-item pairs (i.e., instances
28 * of {@lin PcapPacketPair} where {@link PcapPacketPair#getSecond()} is {@code null}).
30 * @param conv The {@code Conversation} for which packet pairs are to be extracted.
31 * @return The packet pairs extracted from {@code conv}.
33 public static List<PcapPacketPair> extractPacketPairs(Conversation conv) {
34 List<PcapPacket> packets = conv.getPackets();
35 List<PcapPacketPair> pairs = new ArrayList<>();
37 while (i < packets.size()) {
38 PcapPacket p1 = packets.get(i);
39 String p1SrcIp = p1.get(IpV4Packet.class).getHeader().getSrcAddr().getHostAddress();
40 int p1SrcPort = p1.get(TcpPacket.class).getHeader().getSrcPort().valueAsInt();
41 if (i+1 < packets.size()) {
42 PcapPacket p2 = packets.get(i+1);
43 if (PcapPacketUtils.isSource(p2, p1SrcIp, p1SrcPort)) {
44 // Two packets in a row going in the same direction -> create one item pair for p1
45 pairs.add(new PcapPacketPair(p1, null));
46 // Advance one packet as the following two packets may form a valid two-item pair.
49 // The two packets form a response-reply pair, create two-item pair.
50 pairs.add(new PcapPacketPair(p1, p2));
51 // Advance two packets as we have already processed the packet at index i+1 in order to create the pair.
55 // Last packet of conversation => one item pair
56 pairs.add(new PcapPacketPair(p1, null));
57 // Advance i to ensure termination.
62 // TODO: what if there is long time between response and reply packet? Should we add a threshold and exclude those cases?
66 * Given a collection of TCP conversations and associated DNS mappings, groups the conversations by hostname.
67 * @param tcpConversations The collection of TCP conversations.
68 * @param ipHostnameMappings The associated DNS mappings.
69 * @return A map where each key is a hostname and its associated value is a list of conversations where one of the
70 * two communicating hosts is that hostname (i.e. its IP maps to the hostname).
72 public static Map<String, List<Conversation>> groupConversationsByHostname(Collection<Conversation> tcpConversations, DnsMap ipHostnameMappings) {
73 HashMap<String, List<Conversation>> result = new HashMap<>();
74 for (Conversation c : tcpConversations) {
75 if (c.getPackets().size() == 0) {
76 String warningStr = String.format("Detected a %s [%s] with no payload packets.",
77 c.getClass().getSimpleName(), c.toString());
78 System.err.println(warningStr);
81 IpV4Packet firstPacketIp = c.getPackets().get(0).get(IpV4Packet.class);
82 String ipSrc = firstPacketIp.getHeader().getSrcAddr().getHostAddress();
83 String ipDst = firstPacketIp.getHeader().getDstAddr().getHostAddress();
84 // Check if src or dst IP is associated with one or more hostnames.
85 Set<String> hostnames = ipHostnameMappings.getHostnamesForIp(ipSrc);
86 if (hostnames == null) {
87 // No luck with src ip (possibly because it's a client->srv packet), try dst ip.
88 hostnames = ipHostnameMappings.getHostnamesForIp(ipDst);
90 if (hostnames != null) {
91 // Put a reference to the conversation for each of the hostnames that the conversation's IP maps to.
92 for (String hostname : hostnames) {
93 List<Conversation> newValue = new ArrayList<>();
95 result.merge(hostname, newValue, (l1, l2) -> { l1.addAll(l2); return l1; });
97 if (hostnames.size() > 1) {
98 // Print notice of IP mapping to multiple hostnames (debugging)
99 System.err.println(String.format("%s: encountered an IP that maps to multiple (%d) hostnames",
100 TcpConversationUtils.class.getSimpleName(), hostnames.size()));
103 // If no hostname mapping, store conversation under the key that is the concatenation of the two IPs.
104 // In order to ensure consistency when mapping conversations, use lexicographic order to select which IP
106 String delimiter = "_";
107 // Note that the in case the comparison returns 0, the strings are equal, so it doesn't matter which of
108 // ipSrc and ipDst go first (also, this case should not occur in practice as it means that the device is
109 // communicating with itself!)
110 String key = ipSrc.compareTo(ipDst) <= 0 ? ipSrc + delimiter + ipDst : ipDst + delimiter + ipSrc;
111 List<Conversation> newValue = new ArrayList<>();
113 result.merge(key, newValue, (l1, l2) -> { l1.addAll(l2); return l1; });
119 public static Map<String, Integer> countPacketSequenceFrequencies(Collection<Conversation> conversations) {
120 Map<String, Integer> result = new HashMap<>();
121 for (Conversation conv : conversations) {
122 if (conv.getPackets().size() == 0) {
123 // Skip conversations with no payload packets.
126 StringBuilder sb = new StringBuilder();
127 for (PcapPacket pp : conv.getPackets()) {
128 sb.append(pp.length() + " ");
130 result.merge(sb.toString(), 1, (i1, i2) -> i1+i2);
136 * Given a {@link Collection} of {@link Conversation}s, builds a {@link Map} from {@link String} to {@link List}
137 * of {@link Conversation}s such that each key is the <em>concatenation of the packet lengths of all payload packets
138 * (i.e., the set of packets returned by {@link Conversation#getPackets()}) separated by a delimiter</em> of any
139 * {@link Conversation} pointed to by that key. In other words, what the {@link Conversation}s {@code cs} pointed to
140 * by the key {@code s} have in common is that they all contain exactly the same number of payload packets <em>and
141 * </em> these payload packets are identical across all {@code Conversation}s in {@code convs} in terms of packet
142 * length and packet order. For example, if the key is "152 440 550", this means that every individual
143 * {@code Conversation} in the list of {@code Conversation}s pointed to by that key contain exactly three payload
144 * packet of lengths 152, 440, and 550, and these three packets are ordered in the order prescribed by the key.
145 * This verbose version prints out the SYNACK, SYN, FINACK, FIN, RST, etc. packets.
147 * @param conversations The collection of {@code Conversation}s to group by packet sequence.
148 * @return a {@link Map} from {@link String} to {@link List} of {@link Conversation}s such that each key is the
149 * <em>concatenation of the packet lengths of all payload packets (i.e., the set of packets returned by
150 * {@link Conversation#getPackets()}) separated by a delimiter</em> of any {@link Conversation} pointed to
153 public static Map<String, List<Conversation>> groupConversationsByPacketSequenceVerbose(Collection<Conversation> conversations) {
154 Map<String, List<Conversation>> result = new HashMap<>();
155 for (Conversation conv : conversations) {
156 if (conv.getPackets().size() == 0) {
157 // Skip conversations with no payload packets.
160 StringBuilder sb = new StringBuilder();
161 // Add SYN and SYNACK at front of sequence to indicate if we saw the handshake or if recording started in
162 // the middle of the conversation.
163 for (PcapPacket syn : conv.getSynPackets()) {
164 TcpPacket.TcpHeader tcpHeader = syn.get(TcpPacket.class).getHeader();
165 if (tcpHeader.getSyn() && tcpHeader.getAck()) {
166 // Only append a space if there's preceding content.
167 appendSpaceIfNotEmpty(sb);
169 } else if (tcpHeader.getSyn()) {
170 if (sb.length() != 0) {
171 // If present in the trace, the client's SYN should be at the front of the list, so it should be
172 // appended as the first item.
173 throw new AssertionError("StringBuilder had content when appending SYN");
178 // Then append the length of all application data packets.
179 for (PcapPacket pp : conv.getPackets()) {
180 // Only append a space if there's preceding content.
181 appendSpaceIfNotEmpty(sb);
182 sb.append("(" + conv.getDirection(pp).toCompactString() + "_" + pp.length() + ")");
184 // Then append the logged FINs to indicate if conversation was terminated gracefully.
185 for (FinAckPair fap : conv.getFinAckPairs()) {
186 appendSpaceIfNotEmpty(sb);
187 sb.append(fap.isAcknowledged() ? "FINACK" : "FIN");
189 // Then append the logged RSTs to indicate if conversation was terminated abruptly.
190 for (PcapPacket pp : conv.getRstPackets()) {
191 appendSpaceIfNotEmpty(sb);
194 List<Conversation> oneItemList = new ArrayList<>();
195 oneItemList.add(conv);
196 result.merge(sb.toString(), oneItemList, (oldList, newList) -> {
197 oldList.addAll(newList);
205 * Given a {@link Collection} of {@link Conversation}s, builds a {@link Map} from {@link String} to {@link List}
206 * of {@link Conversation}s such that each key is the <em>concatenation of the packet lengths of all payload packets
207 * (i.e., the set of packets returned by {@link Conversation#getPackets()}) separated by a delimiter</em> of any
208 * {@link Conversation} pointed to by that key. In other words, what the {@link Conversation}s {@code cs} pointed to
209 * by the key {@code s} have in common is that they all contain exactly the same number of payload packets <em>and
210 * </em> these payload packets are identical across all {@code Conversation}s in {@code convs} in terms of packet
211 * length and packet order. For example, if the key is "152 440 550", this means that every individual
212 * {@code Conversation} in the list of {@code Conversation}s pointed to by that key contain exactly three payload
213 * packet of lengths 152, 440, and 550, and these three packets are ordered in the order prescribed by the key.
215 * @param conversations The collection of {@code Conversation}s to group by packet sequence.
216 * @return a {@link Map} from {@link String} to {@link List} of {@link Conversation}s such that each key is the
217 * <em>concatenation of the packet lengths of all payload packets (i.e., the set of packets returned by
218 * {@link Conversation#getPackets()}) separated by a delimiter</em> of any {@link Conversation} pointed to
221 public static Map<String, List<Conversation>> groupConversationsByPacketSequence(Collection<Conversation> conversations) {
222 Map<String, List<Conversation>> result = new HashMap<>();
223 for (Conversation conv : conversations) {
224 if (conv.getPackets().size() == 0) {
225 // Skip conversations with no payload packets.
228 StringBuilder sb = new StringBuilder();
229 // Then append the length of all application data packets.
230 for (PcapPacket pp : conv.getPackets()) {
231 // Only append a space if there's preceding content.
232 appendSpaceIfNotEmpty(sb);
233 sb.append(pp.length());
235 List<Conversation> oneItemList = new ArrayList<>();
236 oneItemList.add(conv);
237 result.merge(sb.toString(), oneItemList, (oldList, newList) -> {
238 oldList.addAll(newList);
246 * Given a {@link Conversation}, counts the frequencies of each unique packet length seen as part of the
247 * {@code Conversation}.
248 * @param c The {@code Conversation} for which unique packet length frequencies are to be determined.
249 * @return A mapping from packet length to its frequency.
251 public static Map<Integer, Integer> countPacketLengthFrequencies(Conversation c) {
252 Map<Integer, Integer> result = new HashMap<>();
253 for (PcapPacket packet : c.getPackets()) {
254 result.merge(packet.length(), 1, (i1, i2) -> i1 + i2);
260 * Like {@link #countPacketLengthFrequencies(Conversation)}, but counts packet length frequencies for a collection
261 * of {@code Conversation}s, i.e., the frequency of a packet length becomes the total number of packets with that
262 * length across <em>all</em> {@code Conversation}s in {@code conversations}.
263 * @param conversations The collection of {@code Conversation}s for which packet length frequencies are to be
265 * @return A mapping from packet length to its frequency.
267 public static Map<Integer, Integer> countPacketLengthFrequencies(Collection<Conversation> conversations) {
268 Map<Integer, Integer> result = new HashMap<>();
269 for (Conversation c : conversations) {
270 Map<Integer, Integer> intermediateResult = countPacketLengthFrequencies(c);
271 for (Map.Entry<Integer, Integer> entry : intermediateResult.entrySet()) {
272 result.merge(entry.getKey(), entry.getValue(), (i1, i2) -> i1 + i2);
278 public static Map<String, Integer> countPacketPairFrequencies(Collection<PcapPacketPair> pairs) {
279 Map<String, Integer> result = new HashMap<>();
280 for (PcapPacketPair ppp : pairs) {
281 result.merge(ppp.toString(), 1, (i1, i2) -> i1 + i2);
286 public static Map<String, Map<String, Integer>> countPacketPairFrequenciesByHostname(Collection<Conversation> tcpConversations, DnsMap ipHostnameMappings) {
287 Map<String, List<Conversation>> convsByHostname = groupConversationsByHostname(tcpConversations, ipHostnameMappings);
288 HashMap<String, Map<String, Integer>> result = new HashMap<>();
289 for (Map.Entry<String, List<Conversation>> entry : convsByHostname.entrySet()) {
290 // Merge all packet pairs exchanged during the course of all conversations with hostname into one list
291 List<PcapPacketPair> allPairsExchangedWithHostname = new ArrayList<>();
292 entry.getValue().forEach(conversation -> allPairsExchangedWithHostname.addAll(extractPacketPairs(conversation)));
293 // Then count the frequencies of packet pairs exchanged with the hostname, irrespective of individual
295 result.put(entry.getKey(), countPacketPairFrequencies(allPairsExchangedWithHostname));
301 * Given a {@link Conversation}, extract its packet length sequence.
302 * @param c The {@link Conversation} from which a packet length sequence is to be extracted.
303 * @return An {@code Integer[]} that holds the packet lengths of all payload-carrying packets in {@code c}. The
304 * packet lengths in the returned array are ordered by packet timestamp.
306 public static Integer[] getPacketLengthSequence(Conversation c) {
307 List<PcapPacket> packets = c.getPackets();
308 Integer[] packetLengthSequence = new Integer[packets.size()];
309 for (int i = 0; i < packetLengthSequence.length; i++) {
310 packetLengthSequence[i] = packets.get(i).getOriginalLength();
312 return packetLengthSequence;
316 * Appends a space to {@code sb} <em>iff</em> {@code sb} already contains some content.
317 * @param sb A {@link StringBuilder} that should have a space appended <em>iff</em> it is not empty.
319 private static void appendSpaceIfNotEmpty(StringBuilder sb) {
320 if (sb.length() != 0) {