public boolean isRelatedToCloudServer(String address, String hostname) {
return ipToHostnameMap.getOrDefault(address, EMPTY_SET).contains(hostname);
}
+
+ public Set<String> getHostnamesForIp(String ip) {
+ Set<String> hostnames = ipToHostnameMap.get(ip);
+ return hostnames != null ? Collections.unmodifiableSet(hostnames) : null;
+ }
}
import java.io.EOFException;
import java.net.UnknownHostException;
import java.time.Instant;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
+import java.util.*;
import java.util.concurrent.TimeoutException;
/**
// -------- 07-19-2018 --------
TriggerTimesFileReader ttfr = new TriggerTimesFileReader();
- List<Instant> triggerTimes = ttfr.readTriggerTimes("/Users/varmarken/Downloads/tplink-feb-13-2018.timestamps", false);
-// triggerTimes.stream().forEach(i -> System.out.println(i.atZone(TriggerTimesFileReader.ZONE_ID_LOS_ANGELES).toString()));
- String pcapFile = "/Users/varmarken/Development/Repositories/UCI/NetworkingGroup/smart_home_traffic/Code/Projects/SmartPlugDetector/pcap/wlan1.local.dns.pcap";
+// List<Instant> triggerTimes = ttfr.readTriggerTimes("/Users/varmarken/Downloads/tplink-feb-13-2018.timestamps", false);
+ List<Instant> triggerTimes = ttfr.readTriggerTimes("/Users/varmarken/temp/UCI IoT Project/June2018 experiments/tplink/tplink-june-14-2018-timestamps.txt", false);
+// String pcapFile = "/Users/varmarken/Development/Repositories/UCI/NetworkingGroup/smart_home_traffic/Code/Projects/SmartPlugDetector/pcap/wlan1.local.dns.pcap";
+ String pcapFile = "/Users/varmarken/temp/UCI IoT Project/June2018 experiments/tplink/tplink.wlan1.local.pcap";
String tpLinkPlugIp = "192.168.1.159";
TriggerTrafficExtractor tte = new TriggerTrafficExtractor(pcapFile, triggerTimes, tpLinkPlugIp);
- final PcapDumper outputter = Pcaps.openDead(DataLinkType.EN10MB, 65536).dumpOpen("/Users/varmarken/temp/traces/output/tplink-filtered.pcap");
+// final PcapDumper outputter = Pcaps.openDead(DataLinkType.EN10MB, 65536).dumpOpen("/Users/varmarken/temp/traces/output/tplink-filtered.pcap");
+ final PcapDumper outputter = Pcaps.openDead(DataLinkType.EN10MB, 65536).dumpOpen("/Users/varmarken/temp/UCI IoT Project/June2018 experiments/tplink/tplink-filtered.pcap");
DnsMap dnsMap = new DnsMap();
TcpReassembler tcpReassembler = new TcpReassembler();
tte.performExtraction(pkt -> {
outputter.flush();
outputter.close();
+ /*
int packets = 0;
for (Conversation c : tcpReassembler.getTcpConversations()) {
packets += c.getPackets().size();
// Applying filter: "(tcp and not tcp.len == 0 and not tcp.analysis.retransmission and not tcp.analysis.fast_retransmission) or (tcp.flags.syn == 1) or (tcp.flags.fin == 1)"
// to the file gives 295 packets, but there are 24 TCP-Out-Of-Order SYN/SYNACKs which are filtered as retransmissions in Conversation, so the numbers seem to match.
System.out.println("number of packets: " + packets);
+ */
List<List<PcapPacketPair>> pairs = new ArrayList<>();
for (Conversation c : tcpReassembler.getTcpConversations()) {
pairs.add(TcpConversationUtils.extractPacketPairs(c));
}
+ /*
// Sort pairs according to timestamp of first packet of conversation for (debugging) convenience.
Collections.sort(pairs, (l1, l2) -> {
if (l1.get(0).getFirst().getTimestamp().isBefore(l2.get(0).getFirst().getTimestamp())) return -1;
else if (l2.get(0).getFirst().getTimestamp().isBefore(l1.get(0).getFirst().getTimestamp())) return 1;
else return 0;
});
+ */
System.out.println("list of pairs produced");
List<PcapPacketPair> eventstplinkraPairs = new ArrayList<>();
List<List<PcapPacketPair>> otherPairs = new ArrayList<>();
String hostname = "events.tplinkra.com";
+ int emptyLists = 0;
for (List<PcapPacketPair> lppp : pairs) {
+ if (lppp.size() < 1) {
+ emptyLists++;
+ continue;
+ }
IpV4Packet ipPacket = lppp.get(0).getFirst().get(IpV4Packet.class);
// If packets are associated with the hostname
if (dnsMap.isRelatedToCloudServer(ipPacket.getHeader().getSrcAddr().getHostAddress(), hostname) ||
otherPairs.add(lppp);
}
}
+ System.out.println("number of empty list of packet pairs: " + emptyLists);
HashMap<String, Integer> pairCount = new HashMap<>();
for (PcapPacketPair ppp : eventstplinkraPairs) {
if (pairCount.containsKey(ppp.toString())) {
}
}
System.out.println("pairCount map built");
+
+ // Build map containing frequencies of packet lengths exchanged with events.tplinkra.com as well as a map with
+ // the frequencies of specific sequences of packet lengths for the same hostname
+ HashMap<Integer, Integer> eventstplinkraPacketLengthFreqMap = new HashMap<>();
+ HashMap<String, Integer> eventstplinkraPacketSequenceFreqMap = new HashMap<>();
+ for (Conversation c : tcpReassembler.getTcpConversations()) {
+ if (c.getPackets().size() == 0) {
+ continue;
+ }
+ PcapPacket firstPacket = c.getPackets().get(0);
+ IpV4Packet firstPacketIp = firstPacket.get(IpV4Packet.class);
+ if (!dnsMap.isRelatedToCloudServer(firstPacketIp.getHeader().getSrcAddr().getHostAddress(), hostname) &&
+ !dnsMap.isRelatedToCloudServer(firstPacketIp.getHeader().getDstAddr().getHostAddress(), hostname)) {
+ continue;
+ }
+ // Update the packet length freq map
+ for (PcapPacket pp : c.getPackets()) {
+ eventstplinkraPacketLengthFreqMap.merge(pp.length(), 1, (i1, i2) -> i1 + i2);
+ }
+ // Update the packet sequence freq map
+ StringBuilder sb = new StringBuilder();
+ for (PcapPacket pp : c.getPackets()) {
+ sb.append(pp.length() + " ");
+ }
+ eventstplinkraPacketSequenceFreqMap.merge(sb.toString(), 1, (i1, i2) -> i1+i2);
+ }
+ System.out.println("packet length frequency map created");
+
+ Map<String, List<Conversation>> hostnameConversationMap =
+ TcpConversationUtils.groupConversationsByHostname(tcpReassembler.getTcpConversations(), dnsMap);
+ System.out.println("hostnameConversationMap created");
// ----------------------------
}
/**
* Holds <em>terminated</em> {@link Conversation}s.
- * TODO: Should turn this into a list to avoid unintentional overwrite of a Conversation in case ephemeral port number is reused at a later stage...?
*/
- private final Map<Conversation, Conversation> mTerminatedConversations = new HashMap<>();
+ private final List<Conversation> mTerminatedConversations = new ArrayList<>();
@Override
public void gotPacket(PcapPacket pcapPacket) {
*/
public List<Conversation> getTcpConversations() {
ArrayList<Conversation> combined = new ArrayList<>();
- combined.addAll(mTerminatedConversations.values());
+ combined.addAll(mTerminatedConversations);
combined.addAll(mOpenConversations.values());
return combined;
}
// to establish a new conversation with the same four tuple as ongoingConv.
// Mark existing connection as terminated.
// TODO: is this 100% theoretically correct, e.g., if many connection attempts are made back to back? And RST packets?
- mTerminatedConversations.put(ongoingConv, ongoingConv);
+ mTerminatedConversations.add(ongoingConv);
mOpenConversations.remove(ongoingConv);
}
}
private void processRstPacket(PcapPacket rstPacket) {
Conversation conv = getOngoingConversationOrCreateNew(rstPacket);
// Move conversation to set of terminated conversations.
- mTerminatedConversations.put(conv, conv);
+ mTerminatedConversations.add(conv);
mOpenConversations.remove(conv, conv);
}
conv.attemptAcknowledgementOfFin(ackPacket);
if (conv.isGracefullyShutdown()) {
// Move conversation to set of terminated conversations.
- mTerminatedConversations.put(conv, conv);
+ mTerminatedConversations.add(conv);
mOpenConversations.remove(conv);
}
}
package edu.uci.iotproject.analysis;
import edu.uci.iotproject.Conversation;
+import edu.uci.iotproject.DnsMap;
import edu.uci.iotproject.util.PcapPacketUtils;
import org.pcap4j.core.PcapPacket;
import org.pcap4j.packet.IpV4Packet;
import org.pcap4j.packet.TcpPacket;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.*;
/**
* TODO add class documentation.
// TODO: what if there is long time between response and reply packet? Should we add a threshold and exclude those cases?
}
+
+ public static Map<String, List<Conversation>> groupConversationsByHostname(List<Conversation> tcpConversations, DnsMap ipHostnameMappings) {
+ HashMap<String, List<Conversation>> result = new HashMap<>();
+ for (Conversation c : tcpConversations) {
+ if (c.getPackets().size() == 0) {
+ String warningStr = String.format("Detected a %s [%s] with no payload packets.",
+ c.getClass().getSimpleName(), c.toString());
+ System.err.println(warningStr);
+ continue;
+ }
+ IpV4Packet firstPacketIp = c.getPackets().get(0).get(IpV4Packet.class);
+ String ipSrc = firstPacketIp.getHeader().getSrcAddr().getHostAddress();
+ String ipDst = firstPacketIp.getHeader().getDstAddr().getHostAddress();
+ // Check if src or dst IP is associated with one or more hostnames.
+ Set<String> hostnames = ipHostnameMappings.getHostnamesForIp(ipSrc);
+ if (hostnames == null) {
+ // No luck with src ip (possibly because it's a client->srv packet), try dst ip.
+ hostnames = ipHostnameMappings.getHostnamesForIp(ipDst);
+ }
+ if (hostnames != null) {
+ // Put a reference to the conversation for each of the hostnames that the conversation's IP maps to.
+ for (String hostname : hostnames) {
+ List<Conversation> newValue = new ArrayList<>();
+ newValue.add(c);
+ result.merge(hostname, newValue, (l1, l2) -> { l1.addAll(l2); return l1; });
+ }
+ if (hostnames.size() > 1) {
+ // Print notice of IP mapping to multiple hostnames (debugging)
+ System.err.println(String.format("%s: encountered an IP that maps to multiple (%d) hostnames",
+ TcpConversationUtils.class.getSimpleName(), hostnames.size()));
+ }
+ } else {
+ // If no hostname mapping, store conversation under the key that is the concatenation of the two IPs.
+ // In order to ensure consistency when mapping conversations, use lexicographic order to select which IP
+ // goes first.
+ String delimiter = "_";
+ // Note that the in case the comparison returns 0, the strings are equal, so it doesn't matter which of
+ // ipSrc and ipDst go first (also, this case should not occur in practice as it means that the device is
+ // communicating with itself!)
+ String key = ipSrc.compareTo(ipDst) <= 0 ? ipSrc + delimiter + ipDst : ipDst + delimiter + ipSrc;
+ List<Conversation> newValue = new ArrayList<>();
+ newValue.add(c);
+ result.merge(key, newValue, (l1, l2) -> { l1.addAll(l2); return l1; });
+ }
+ }
+ return result;
+ }
+
+
}
* @throws TimeoutException if packets are being read from a live capture and the timeout expired.
*/
public void readFromHandle() throws PcapNativeException, NotOpenException, TimeoutException {
+ int outOfOrderPackets = 0;
try {
PcapPacket prevPacket = null;
PcapPacket packet;
while ((packet = mHandle.getNextPacketEx()) != null) {
if (prevPacket != null && packet.getTimestamp().isBefore(prevPacket.getTimestamp())) {
- System.out.println("Out-of-order (in terms of timestamp) packet detected");
+ outOfOrderPackets++;
/*
// Fail early if assumption doesn't hold.
mHandle.close();
// Reached end of file. All good.
System.out.println(String.format("%s: finished reading pcap file", getClass().getSimpleName()));
}
+ if (outOfOrderPackets > 0) {
+ System.err.println(
+ String.format("[[[ %s: %d packets appeared out of order (with regards to their timestamps) ]]]",
+ getClass().getSimpleName(), outOfOrderPackets));
+ }
mHandle.close();
}
projects / pingpong.git / commitdiff