From: rtrimana Date: Fri, 4 May 2018 21:10:03 +0000 (-0700) Subject: Adding feature to hold multiple hostnames and lists of packet orders in FlowPattern... X-Git-Url: http://plrg.eecs.uci.edu/git/?p=pingpong.git;a=commitdiff_plain;h=ec18f2088211f3e2ef5e10ae79c00e64c8ae5042 Adding feature to hold multiple hostnames and lists of packet orders in FlowPattern, but keeping the old method of searching patterns (i.e. just one hostname and one pattern) for now---need to think more carefully so that we won't break stuff. --- diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FlowPattern.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FlowPattern.java index fb0433c..aca63ab 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FlowPattern.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FlowPattern.java @@ -5,44 +5,24 @@ import org.pcap4j.packet.*; import org.pcap4j.packet.DnsPacket; import org.pcap4j.packet.namednumber.DnsResourceRecordType; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - import java.io.EOFException; +import java.net.InetAddress; import java.net.UnknownHostException; +import java.util.*; import java.util.concurrent.TimeoutException; /** * TODO add class documentation. + * TODO: At this point, this class is still in transition to having multiple hostnames and lists of packets * * @author Janus Varmarken */ public class FlowPattern { - static { - // TP-Link Local ON packet lengths (TCP payload only), extracted from ON event at Feb 13, 2018 13:38:04 - // of the 5 switch data collection: - // 517 1448 1448 1448 855 191 51 490 1027 31 - - ArrayList packetLengths = new ArrayList<>(); - packetLengths.addAll(Arrays.asList(new Integer[] {517, 1448, 1448, 1448, 855, 191, 51, 490, 1027, 31})); - TP_LINK_LOCAL_ON = new FlowPattern("TP_LINK_LOCAL_ON", "events.tplinkra.com", packetLengths); - } - - public static final FlowPattern TP_LINK_LOCAL_ON; - /** * Class properties */ - private final String patternId; - - /** - * The hostname that this {@code FlowPattern} is associated with. - */ + private final String mPatternId; private final String hostname; // The hostname that this {@code FlowPattern} is associated with. /** @@ -50,9 +30,10 @@ public class FlowPattern { * TODO: this is a simplified representation, we should also include information about direction of each packet. */ private final List flowPacketOrder; - - private final Map> hostnameToPacketOrderMap; - private final PcapHandle pcap; + private final Map> mHostnameToPacketLengthsMap; + private final List mHostnameList; + private final PcapHandle mPcap; + /** * Class constants @@ -62,15 +43,17 @@ public class FlowPattern { /** * Constructor #1 */ - public FlowPattern(String patternId, String hostname, PcapHandle pcap) { - this.patternId = patternId; + public FlowPattern(String mPatternId, String hostname, PcapHandle mPcap) { + this.mPatternId = mPatternId; this.hostname = hostname; - this.pcap = pcap; - this.hostnameToPacketOrderMap = null; + this.mHostnameList = null; + this.mPcap = mPcap; + this.mHostnameToPacketLengthsMap = null; this.flowPacketOrder = new ArrayList(); processPcap(); } + /** * Process the PcapHandle to strip off unnecessary packets and just get the integer array of packet lengths */ @@ -78,7 +61,7 @@ public class FlowPattern { PcapPacket packet; try { - while ((packet = pcap.getNextPacketEx()) != null) { + while ((packet = mPcap.getNextPacketEx()) != null) { // For now, we only work support pattern search in TCP over IPv4. IpV4Packet ipPacket = packet.get(IpV4Packet.class); TcpPacket tcpPacket = packet.get(TcpPacket.class); @@ -91,7 +74,7 @@ public class FlowPattern { } } catch (EOFException eofe) { System.out.println("[ FlowPattern ] Finished processing a training PCAP stream!"); - System.out.println("[ FlowPattern ] Pattern for " + patternId + ": " + Arrays.toString(flowPacketOrder.toArray())); + System.out.println("[ FlowPattern ] Pattern for " + mPatternId + ": " + Arrays.toString(flowPacketOrder.toArray())); } catch (PcapNativeException | TimeoutException | NotOpenException ex) { @@ -99,47 +82,119 @@ public class FlowPattern { } } + /** - * Constructor #2 - * - * @param patternId Label for this pattern - * @param hostname Hostname associated with this pattern - * @param flowPacketOrder List of packets in order + * Process the PcapHandle to strip off unnecessary packets. + * We then map list of hostnames to their respective arrays of packet lengths */ - public FlowPattern(String patternId, String hostname, List flowPacketOrder) { - this.patternId = patternId; - this.hostname = hostname; - this.hostnameToPacketOrderMap = null; - this.pcap = null; - this.flowPacketOrder = Collections.unmodifiableList(flowPacketOrder); + private void processPcapToMap() { + + PcapPacket packet; + try { + int hostIndex = -1; + Set addressSet = new HashSet<>(); + while ((packet = mPcap.getNextPacketEx()) != null) { + // For now, we only work support pattern search in TCP over IPv4. + IpV4Packet ipPacket = packet.get(IpV4Packet.class); + TcpPacket tcpPacket = packet.get(TcpPacket.class); + if (ipPacket == null || tcpPacket == null) { + continue; + } + if (tcpPacket.getPayload() == null) { + // We skip non-payload control packets as these are less predictable + continue; + } + // We assume that if it is not a local address then it is a cloud server address + InetAddress srcAddress = ipPacket.getHeader().getSrcAddr(); + InetAddress dstAddress = ipPacket.getHeader().getDstAddr(); + boolean fromServer = !srcAddress.isSiteLocalAddress(); + boolean fromClient = !dstAddress.isSiteLocalAddress(); + if (!fromServer && !fromClient) { + // Packet not related to pattern, skip it + continue; + } else { + // We relate and assume that this address is from our cloud server + String cloudAddress = null; + if (fromClient) { + cloudAddress = dstAddress.getHostAddress(); + } else { // fromServer + cloudAddress = srcAddress.getHostAddress(); + } + //System.out.println("\nCloud address: " + cloudAddress); + if (!addressSet.contains(cloudAddress)) { + addressSet.add(cloudAddress); + hostIndex++; + } + + String hostname = mHostnameList.get(hostIndex); + List packetLengthsList = mHostnameToPacketLengthsMap.containsKey(hostname) ? + mHostnameToPacketLengthsMap.get(hostname) : new ArrayList<>(); + int packetLength = tcpPacket.getPayload().length(); + packetLengthsList.add(packetLength); + mHostnameToPacketLengthsMap.put(hostname, packetLengthsList); + } + } + } catch (EOFException eofe) { + System.out.println("[ FlowPattern ] Finished processing a training PCAP stream!"); + System.out.println("[ FlowPattern ] Pattern for " + mPatternId + ": " + Arrays.toString(mHostnameToPacketLengthsMap.entrySet().toArray())); + } catch (PcapNativeException | + TimeoutException | + NotOpenException ex) { + ex.printStackTrace(); + } } + /** - * Constructor #3 + * Constructor #2 */ - public FlowPattern(String patternId, String hostname, Map> hostnameToPacketOrderMap) { - this.patternId = patternId; - this.hostname = hostname; - this.pcap = null; + public FlowPattern(String mPatternId, List mHostnameList, PcapHandle mPcap) { + this.mPatternId = mPatternId; + this.hostname = null; + this.mHostnameList = mHostnameList; + this.mPcap = mPcap; this.flowPacketOrder = null; - this.hostnameToPacketOrderMap = Collections.unmodifiableMap(hostnameToPacketOrderMap); + this.mHostnameToPacketLengthsMap = new HashMap<>(); + processPcapToMap(); } + public String getPatternId() { - return patternId; + return mPatternId; } + public String getHostname() { return hostname; } + /** - * Get the the sequence of packet lengths that defines this {@code FlowPattern}. + * Get the sequence of packet lengths that defines this {@code FlowPattern}. * @return the sequence of packet lengths that defines this {@code FlowPattern}. */ public List getPacketOrder() { return flowPacketOrder; } + + + /** + * Get the sequence of packet lengths based on input hostname. + * @return the sequence of packet lengths that defines this {@code FlowPattern}. + */ + public List getPacketOrder(String hostname) { + return mHostnameToPacketLengthsMap.get(hostname); + } + + + /** + * Get the list of associated hostnames. + * @return the associated hostnames that define this {@code FlowPattern}. + */ + public List getHostnameList() { + return mHostnameList; + } + /** * Get the length of the List of {@code FlowPattern}. @@ -147,6 +202,14 @@ public class FlowPattern { */ public int getLength() { return flowPacketOrder.size(); - } + } + + /** + * Get the length of the List of {@code FlowPattern}. + * @return the length of the List of {@code FlowPattern}. + */ + public int getLength(String hostname) { + return mHostnameToPacketLengthsMap.get(hostname).size(); + } } diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FlowPatternFinder.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FlowPatternFinder.java index 1152946..da0d3eb 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FlowPatternFinder.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/FlowPatternFinder.java @@ -90,6 +90,10 @@ public class FlowPatternFinder { private void findFlowPattern() { try { PcapPacket packet; +// TODO: The new comparison method is pending +// TODO: For now, just compare using one hostname and one list per FlowPattern +// List hostnameList = mPattern.getHostnameList(); +// int hostIndex = 0; int patternLength = mPattern.getLength(); while ((packet = mPcap.getNextPacketEx()) != null) { // Let DnsMap handle DNS packets. @@ -104,6 +108,10 @@ public class FlowPatternFinder { if (ipPacket == null || tcpPacket == null) { continue; } + if (tcpPacket.getPayload() == null) { + // We skip non-payload control packets as these are less predictable + continue; + } String srcAddress = ipPacket.getHeader().getSrcAddr().getHostAddress(); String dstAddress = ipPacket.getHeader().getDstAddr().getHostAddress(); int srcPort = tcpPacket.getHeader().getSrcPort().valueAsInt(); @@ -111,14 +119,13 @@ public class FlowPatternFinder { // Is this packet related to the pattern; i.e. is it going to (or coming from) the cloud server? boolean fromServer = mDnsMap.isRelatedToCloudServer(srcAddress, mPattern.getHostname()); boolean fromClient = mDnsMap.isRelatedToCloudServer(dstAddress, mPattern.getHostname()); +// String currentHostname = hostnameList.get(hostIndex); +// boolean fromServer = mDnsMap.isRelatedToCloudServer(srcAddress, currentHostname); +// boolean fromClient = mDnsMap.isRelatedToCloudServer(dstAddress, currentHostname); if (!fromServer && !fromClient) { // Packet not related to pattern, skip it. continue; } - if (tcpPacket.getPayload() == null) { - // We skip non-payload control packets as these are less predictable - continue; - } // Conversations (connections/sessions) are identified by the four-tuple // (clientIp, clientPort, serverIp, serverPort) (see Conversation Javadoc). // Create "dummy" conversation for looking up an existing entry. @@ -138,6 +145,7 @@ public class FlowPatternFinder { // Refresh reference to point to entry in map (in case packet was added to existing entry). conversation = mConversations.get(conversation); if (conversation.getPackets().size() == mPattern.getLength()) { +// if (conversation.getPackets().size() == mPattern.getLength(currentHostname)) { // Conversation reached a size that matches the expected size. // Remove the Conversation from the map and start the analysis. // Any future packets identified by the same four tuple will be tied to a new Conversation instance. @@ -147,6 +155,8 @@ public class FlowPatternFinder { PatternComparisonTask comparisonTask = new PatternComparisonTask<>(conversation, mPattern, ComparisonFunctions.COMPLETE_MATCH); mPendingComparisons.add(EXECUTOR_SERVICE.submit(comparisonTask)); + // Increment hostIndex to find the next + } } } catch (EOFException eofe) { diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java index 239d8c1..b828323 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/Main.java @@ -4,6 +4,7 @@ import org.pcap4j.core.*; import java.io.EOFException; import java.net.UnknownHostException; +import java.util.*; import java.util.concurrent.TimeoutException; /** @@ -21,7 +22,8 @@ public class Main { public static void main(String[] args) throws PcapNativeException, NotOpenException, EOFException, TimeoutException, UnknownHostException { final String fileName = args.length > 0 ? args[0] : "/home/rtrimana/pcap_processing/smart_home_traffic/Code/Projects/SmartPlugDetector/pcap/wlan1.local.remote.dns.pcap"; - final String trainingFileName = "./pcap/TP_LINK_LOCAL_OFF.pcap"; + final String trainingFileName = "./pcap/TP_LINK_LOCAL_ON.pcap"; + //final String trainingFileName = "./pcap/TP_LINK_REMOTE_ON.pcap"; // ====== Debug code ====== PcapHandle handle; @@ -33,9 +35,17 @@ public class Main { handle = Pcaps.openOffline(fileName); trainingPcap = Pcaps.openOffline(trainingFileName); } - FlowPattern fp = new FlowPattern("TP_LINK_LOCAL_OFF", "events.tplinkra.com", trainingPcap); - - //FlowPatternFinder fpf = new FlowPatternFinder(handle, FlowPattern.TP_LINK_LOCAL_ON); + + // TODO: The followings are the way to extract multiple hostnames and their associated packet lengths lists + //List list = new ArrayList<>(); + //list.add("events.tplinkra.com"); + //FlowPattern fp = new FlowPattern("TP_LINK_LOCAL_ON", list, trainingPcap); + //List list2 = new ArrayList<>(); + //list2.add("devs.tplinkcloud.com"); + //list2.add("events.tplinkra.com"); + //FlowPattern fp3 = new FlowPattern("TP_LINK_REMOTE_ON", list2, trainingPcap); + + FlowPattern fp = new FlowPattern("TP_LINK_LOCAL_ON", "events.tplinkra.com", trainingPcap); FlowPatternFinder fpf = new FlowPatternFinder(handle, fp); fpf.start(); diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/PatternComparisonTask.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/PatternComparisonTask.java index 705646c..808dee7 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/PatternComparisonTask.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/comparison/PatternComparisonTask.java @@ -17,7 +17,7 @@ public class PatternComparisonTask> private final Conversation mConversation; private final FlowPattern mFlowPattern; - private final BiFunction mComparitor; + private final BiFunction mComparator; /** * Create a new {@code PatternComparisonTask}. @@ -29,12 +29,12 @@ public class PatternComparisonTask> public PatternComparisonTask(Conversation conversation, FlowPattern pattern, BiFunction comparisonFunction) { this.mConversation = conversation; this.mFlowPattern = pattern; - this.mComparitor = comparisonFunction; + this.mComparator = comparisonFunction; } @Override public R call() throws Exception { - return mComparitor.apply(mConversation, mFlowPattern); + return mComparator.apply(mConversation, mFlowPattern); } -} \ No newline at end of file +}