+ */
+
+ // ================================================ CLUSTERING ================================================
+ // Note: no need to use the more convoluted on/off maps; can simply use the UserAction->List<Conversation> map
+ // when don't care about hostnames and sequences (see comment earlier).
+ List<Conversation> onConversations = userActionToConversations.entrySet().stream().
+ filter(e -> e.getKey().getType() == Type.TOGGLE_ON). // drop all OFF events from stream
+ map(e -> e.getValue()). // no longer interested in the UserActions
+ flatMap(List::stream). // flatten List<List<T>> to a List<T>
+ collect(Collectors.toList());
+ List<Conversation> offConversations = userActionToConversations.entrySet().stream().
+ filter(e -> e.getKey().getType() == Type.TOGGLE_OFF).
+ map(e -> e.getValue()).
+ flatMap(List::stream).
+ collect(Collectors.toList());
+ //Collections.sort(onConversations, (c1, c2) -> c1.getPackets().)
+
+ List<PcapPacketPair> onPairs = onConversations.stream().
+ map(c -> c.isTls() ? TcpConversationUtils.extractTlsAppDataPacketPairs(c) :
+ TcpConversationUtils.extractPacketPairs(c)).
+ flatMap(List::stream). // flatten List<List<>> to List<>
+ collect(Collectors.toList());
+ List<PcapPacketPair> offPairs = offConversations.stream().
+ map(c -> c.isTls() ? TcpConversationUtils.extractTlsAppDataPacketPairs(c) :
+ TcpConversationUtils.extractPacketPairs(c)).
+ flatMap(List::stream). // flatten List<List<>> to List<>
+ collect(Collectors.toList());
+ // Note: need to update the DnsMap of all PcapPacketPairs if we want to use the IP/hostname-sensitive distance.
+ Stream.concat(Stream.of(onPairs), Stream.of(offPairs)).flatMap(List::stream).forEach(p -> p.setDnsMap(dnsMap));
+ // Perform clustering on conversation logged as part of all ON events.
+ DBSCANClusterer<PcapPacketPair> onClusterer = new DBSCANClusterer<>(10.0, 45);
+ List<Cluster<PcapPacketPair>> onClusters = onClusterer.cluster(onPairs);
+ // Perform clustering on conversation logged as part of all OFF events.
+ DBSCANClusterer<PcapPacketPair> offClusterer = new DBSCANClusterer<>(10.0, 45);
+ List<Cluster<PcapPacketPair>> offClusters = offClusterer.cluster(offPairs);
+ // Sort the conversations as reference
+ List<Conversation> sortedAllConversation = TcpConversationUtils.sortConversationList(allConversations);
+ // Output clusters
+ System.out.println("========================================");
+ System.out.println(" Clustering results for ON ");
+ System.out.println(" Number of clusters: " + onClusters.size());
+ int count = 0;
+ List<List<List<PcapPacket>>> ppListOfListReadOn = new ArrayList<>();
+ List<List<List<PcapPacket>>> ppListOfListListOn = new ArrayList<>();
+ for (Cluster<PcapPacketPair> c : onClusters) {
+ System.out.println(String.format("<<< Cluster #%02d (%03d points) >>>", ++count, c.getPoints().size()));
+ System.out.print(PrintUtils.toSummaryString(c));
+ if(c.getPoints().size() > 45 && c.getPoints().size() < 55) {
+ // Print to file
+ List<List<PcapPacket>> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c);
+ ppListOfListListOn.add(ppListOfList);
+ }
+ }
+ // TODO: Merging test
+ ppListOfListListOn = PcapPacketUtils.mergeSignatures(ppListOfListListOn, sortedAllConversation);
+ ppListOfListListOn = PcapPacketUtils.sortSignatures(ppListOfListListOn);
+ // TODO: Need to remove sequence number 2 for ST plug since it is not a good signature!
+ //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 2);
+ // TODO: Need to remove sequence number 0 for TP-Link plug since it is not a good signature!
+ // TODO: This sequence actually belongs to the local communication between the plug and the phone
+ //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 0);
+ PcapPacketUtils.printSignatures(ppListOfListListOn);
+ //count = 0;
+ /*for (List<List<PcapPacket>> ll : ppListOfListListOn) {
+ PrintUtils.serializeClustersIntoFile("./onSignature" + ++count + ".sig", ll);
+ ppListOfListReadOn.add(PrintUtils.deserializeClustersFromFile("./onSignature" + count + ".sig"));
+ }*/
+ PrintUtils.serializeSignatureIntoFile("./onSignature.sig", ppListOfListListOn);
+ ppListOfListReadOn = PrintUtils.deserializeSignatureFromFile("./onSignature.sig");
+
+ System.out.println("========================================");
+ System.out.println(" Clustering results for OFF ");
+ System.out.println(" Number of clusters: " + offClusters.size());
+ count = 0;
+ List<List<List<PcapPacket>>> ppListOfListReadOff = new ArrayList<>();
+ List<List<List<PcapPacket>>> ppListOfListListOff = new ArrayList<>();
+ for (Cluster<PcapPacketPair> c : offClusters) {
+ System.out.println(String.format("<<< Cluster #%03d (%06d points) >>>", ++count, c.getPoints().size()));
+ System.out.print(PrintUtils.toSummaryString(c));
+ if(c.getPoints().size() > 45 && c.getPoints().size() < 55) {
+ // Print to file
+ List<List<PcapPacket>> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c);
+ ppListOfListListOff.add(ppListOfList);
+ }
+ }
+ // TODO: Merging test
+ ppListOfListListOff = PcapPacketUtils.mergeSignatures(ppListOfListListOff, sortedAllConversation);
+ ppListOfListListOff = PcapPacketUtils.sortSignatures(ppListOfListListOff);
+ // TODO: Need to remove sequence number 1 for Nest Thermostat since it is not a good signature!
+ //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 1);
+ // TODO: Need to remove sequence number 0 for Arlo Camera since it is not a good signature!
+ //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 0);
+ // TODO: Need to remove sequence number 2 for ST plug since it is not a good signature!
+ //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 2);
+ // TODO: Need to remove sequence number 0 for TP-Link plug since it is not a good signature!
+ // TODO: This sequence actually belongs to the local communication between the plug and the phone
+ //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 0);
+ PcapPacketUtils.printSignatures(ppListOfListListOff);
+ //count = 0;
+ /*for (List<List<PcapPacket>> ll : ppListOfListListOff) {
+ PrintUtils.serializeClustersIntoFile("./offSignature" + ++count + ".sig", ll);
+ ppListOfListReadOff.add(PrintUtils.deserializeClustersFromFile("./offSignature" + count + ".sig"));
+ }*/
+ PrintUtils.serializeSignatureIntoFile("./offSignature.sig", ppListOfListListOff);
+ ppListOfListReadOff = PrintUtils.deserializeSignatureFromFile("./offSignature.sig");
+ System.out.println("========================================");
+ // ============================================================================================================