- List<Conversation> onConversations = userActionToConversations.entrySet().stream().
- filter(e -> e.getKey().getType() == Type.TOGGLE_ON). // drop all OFF events from stream
- map(e -> e.getValue()). // no longer interested in the UserActions
- flatMap(List::stream). // flatten List<List<T>> to a List<T>
- collect(Collectors.toList());
- List<Conversation> offConversations = userActionToConversations.entrySet().stream().
- filter(e -> e.getKey().getType() == Type.TOGGLE_OFF).
- map(e -> e.getValue()).
- flatMap(List::stream).
- collect(Collectors.toList());
- //Collections.sort(onConversations, (c1, c2) -> c1.getPackets().)
-
- List<PcapPacketPair> onPairs = onConversations.stream().
- map(c -> c.isTls() ? TcpConversationUtils.extractTlsAppDataPacketPairs(c) :
- TcpConversationUtils.extractPacketPairs(c)).
- flatMap(List::stream). // flatten List<List<>> to List<>
- collect(Collectors.toList());
- List<PcapPacketPair> offPairs = offConversations.stream().
- map(c -> c.isTls() ? TcpConversationUtils.extractTlsAppDataPacketPairs(c) :
- TcpConversationUtils.extractPacketPairs(c)).
- flatMap(List::stream). // flatten List<List<>> to List<>
- collect(Collectors.toList());
- // Note: need to update the DnsMap of all PcapPacketPairs if we want to use the IP/hostname-sensitive distance.
- Stream.concat(Stream.of(onPairs), Stream.of(offPairs)).flatMap(List::stream).forEach(p -> p.setDnsMap(dnsMap));
- // Perform clustering on conversation logged as part of all ON events.
- DBSCANClusterer<PcapPacketPair> onClusterer = new DBSCANClusterer<>(10.0, 45);
- //DBSCANClusterer<PcapPacketPair> onClusterer = new DBSCANClusterer<>(3, 45);
- //DBSCANClusterer<PcapPacketPair> onClusterer = new DBSCANClusterer<>(10.0, 10);
- List<Cluster<PcapPacketPair>> onClusters = onClusterer.cluster(onPairs);
- // Perform clustering on conversation logged as part of all OFF events.
- DBSCANClusterer<PcapPacketPair> offClusterer = new DBSCANClusterer<>(10.0, 45);
- //DBSCANClusterer<PcapPacketPair> offClusterer = new DBSCANClusterer<>(3, 45);
- //DBSCANClusterer<PcapPacketPair> offClusterer = new DBSCANClusterer<>(10.0, 10);
- List<Cluster<PcapPacketPair>> offClusters = offClusterer.cluster(offPairs);
- // Sort the conversations as reference
- List<Conversation> sortedAllConversation = TcpConversationUtils.sortConversationList(allConversations);
- // Output clusters
- System.out.println("========================================");
- System.out.println(" Clustering results for ON ");
- System.out.println(" Number of clusters: " + onClusters.size());
- int count = 0;
- List<List<List<PcapPacket>>> ppListOfListReadOn = new ArrayList<>();
- List<List<List<PcapPacket>>> ppListOfListListOn = new ArrayList<>();
- for (Cluster<PcapPacketPair> c : onClusters) {
- System.out.println(String.format("<<< Cluster #%02d (%03d points) >>>", ++count, c.getPoints().size()));
- System.out.print(PrintUtils.toSummaryString(c));
- if(c.getPoints().size() > 45 && c.getPoints().size() < 55) {
- //if(c.getPoints().size() > 25) {
- // Print to file
- List<List<PcapPacket>> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c);
- ppListOfListListOn.add(ppListOfList);
- }
- }
- // TODO: Merging test
- ppListOfListListOn = PcapPacketUtils.mergeSignatures(ppListOfListListOn, sortedAllConversation);
- // TODO: Need to remove sequence 550 567 for Blossom phone side since it is not a good signature (overlap)!
-// PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 1);
- // TODO: Need to remove sequence 69 296 for Blossom device side since it is not a good signature (overlap)!
-// PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 2);
- // TODO: Need to remove sequence number 2 for ST plug since it is not a good signature!
- //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 2);
- // TODO: Need to remove sequence number 0 for Arlo Camera since it is not a good signature!
- //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 0);
- // TODO: Need to remove sequence number 0 for TP-Link plug since it is not a good signature!
- // TODO: This sequence actually belongs to the local communication between the plug and the phone
- //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 0);
- ppListOfListListOn = PcapPacketUtils.sortSignatures(ppListOfListListOn);
- PcapPacketUtils.printSignatures(ppListOfListListOn);
- //count = 0;
- /*for (List<List<PcapPacket>> ll : ppListOfListListOn) {
- PrintUtils.serializeClustersIntoFile("./onSignature" + ++count + ".sig", ll);
- ppListOfListReadOn.add(PrintUtils.deserializeClustersFromFile("./onSignature" + count + ".sig"));
- }*/
- PrintUtils.serializeSignatureIntoFile("./onSignature.sig", ppListOfListListOn);
- ppListOfListReadOn = PrintUtils.deserializeSignatureFromFile("./onSignature.sig");
-
- System.out.println("========================================");
- System.out.println(" Clustering results for OFF ");
- System.out.println(" Number of clusters: " + offClusters.size());
- count = 0;
- List<List<List<PcapPacket>>> ppListOfListReadOff = new ArrayList<>();
- List<List<List<PcapPacket>>> ppListOfListListOff = new ArrayList<>();
- for (Cluster<PcapPacketPair> c : offClusters) {
- System.out.println(String.format("<<< Cluster #%03d (%06d points) >>>", ++count, c.getPoints().size()));
- System.out.print(PrintUtils.toSummaryString(c));
- if(c.getPoints().size() > 45 && c.getPoints().size() < 55) {
- //if(c.getPoints().size() > 25) {
- // Print to file
- List<List<PcapPacket>> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c);
- ppListOfListListOff.add(ppListOfList);
- }
- }
- // TODO: Merging test
- ppListOfListListOff = PcapPacketUtils.mergeSignatures(ppListOfListListOff, sortedAllConversation);
- // TODO: Need to remove sequence 69 296 for Blossom device side since it is not a good signature (overlap)!
-// PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 3);
- // TODO: Need to remove sequence number 1 for Nest Thermostat since it is not a good signature!
- //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 1);
- // TODO: Need to remove sequence number 0 for Arlo Camera since it is not a good signature!
- PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 1);
- // TODO: Need to remove sequence number 2 for ST plug since it is not a good signature!
- //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 2);
- // TODO: Need to remove sequence number 0 for TP-Link plug since it is not a good signature!
- // TODO: This sequence actually belongs to the local communication between the plug and the phone
- //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 0);
- ppListOfListListOff = PcapPacketUtils.sortSignatures(ppListOfListListOff);
- PcapPacketUtils.printSignatures(ppListOfListListOff);
- //count = 0;
- /*for (List<List<PcapPacket>> ll : ppListOfListListOff) {
- PrintUtils.serializeClustersIntoFile("./offSignature" + ++count + ".sig", ll);
- ppListOfListReadOff.add(PrintUtils.deserializeClustersFromFile("./offSignature" + count + ".sig"));
- }*/
- PrintUtils.serializeSignatureIntoFile("./offSignature.sig", ppListOfListListOff);
- ppListOfListReadOff = PrintUtils.deserializeSignatureFromFile("./offSignature.sig");
- System.out.println("========================================");
+// List<Conversation> onConversations = userActionToConversations.entrySet().stream().
+// filter(e -> e.getKey().getType() == Type.TOGGLE_ON). // drop all OFF events from stream
+// map(e -> e.getValue()). // no longer interested in the UserActions
+// flatMap(List::stream). // flatten List<List<T>> to a List<T>
+// collect(Collectors.toList());
+// List<Conversation> offConversations = userActionToConversations.entrySet().stream().
+// filter(e -> e.getKey().getType() == Type.TOGGLE_OFF).
+// map(e -> e.getValue()).
+// flatMap(List::stream).
+// collect(Collectors.toList());
+// //Collections.sort(onConversations, (c1, c2) -> c1.getPackets().)
+//
+// List<PcapPacketPair> onPairs = onConversations.stream().
+// map(c -> c.isTls() ? TcpConversationUtils.extractTlsAppDataPacketPairs(c) :
+// TcpConversationUtils.extractPacketPairs(c)).
+// flatMap(List::stream). // flatten List<List<>> to List<>
+// collect(Collectors.toList());
+// List<PcapPacketPair> offPairs = offConversations.stream().
+// map(c -> c.isTls() ? TcpConversationUtils.extractTlsAppDataPacketPairs(c) :
+// TcpConversationUtils.extractPacketPairs(c)).
+// flatMap(List::stream). // flatten List<List<>> to List<>
+// collect(Collectors.toList());
+// // Note: need to update the DnsMap of all PcapPacketPairs if we want to use the IP/hostname-sensitive distance.
+// Stream.concat(Stream.of(onPairs), Stream.of(offPairs)).flatMap(List::stream).forEach(p -> p.setDnsMap(dnsMap));
+// // Perform clustering on conversation logged as part of all ON events.
+//// DBSCANClusterer<PcapPacketPair> onClusterer = new DBSCANClusterer<>(10.0, 45);
+// DBSCANClusterer<PcapPacketPair> onClusterer = new DBSCANClusterer<>(2, 2);
+// //DBSCANClusterer<PcapPacketPair> onClusterer = new DBSCANClusterer<>(10.0, 10);
+// List<Cluster<PcapPacketPair>> onClusters = onClusterer.cluster(onPairs);
+// // Perform clustering on conversation logged as part of all OFF events.
+//// DBSCANClusterer<PcapPacketPair> offClusterer = new DBSCANClusterer<>(10.0, 45);
+// DBSCANClusterer<PcapPacketPair> offClusterer = new DBSCANClusterer<>(2, 2);
+// //DBSCANClusterer<PcapPacketPair> offClusterer = new DBSCANClusterer<>(10.0, 10);
+// List<Cluster<PcapPacketPair>> offClusters = offClusterer.cluster(offPairs);
+// // Sort the conversations as reference
+// List<Conversation> sortedAllConversation = TcpConversationUtils.sortConversationList(allConversations);
+// // Output clusters
+// System.out.println("========================================");
+// System.out.println(" Clustering results for ON ");
+// System.out.println(" Number of clusters: " + onClusters.size());
+// int count = 0;
+// List<List<List<PcapPacket>>> ppListOfListReadOn = new ArrayList<>();
+// List<List<List<PcapPacket>>> ppListOfListListOn = new ArrayList<>();
+// for (Cluster<PcapPacketPair> c : onClusters) {
+// System.out.println(String.format("<<< Cluster #%02d (%03d points) >>>", ++count, c.getPoints().size()));
+// System.out.print(PrintUtils.toSummaryString(c));
+// if(c.getPoints().size() > 45 && c.getPoints().size() < 55) {
+// //if(c.getPoints().size() > 25) {
+// // Print to file
+// List<List<PcapPacket>> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c);
+// ppListOfListListOn.add(ppListOfList);
+// }
+// }
+// // TODO: Merging test
+// ppListOfListListOn = PcapPacketUtils.mergeSignatures(ppListOfListListOn, sortedAllConversation);
+// // TODO: Need to remove sequence 550 567 for Blossom phone side since it is not a good signature (overlap)!
+//// PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 1);
+// // TODO: Need to remove sequence 69 296 for Blossom device side since it is not a good signature (overlap)!
+//// PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 2);
+// // TODO: Need to remove sequence number 2 for ST plug since it is not a good signature!
+// //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 2);
+// // TODO: Need to remove sequence number 0 for Arlo Camera since it is not a good signature!
+// //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 0);
+// // TODO: Need to remove sequence number 0 for TP-Link plug since it is not a good signature!
+// // TODO: This sequence actually belongs to the local communication between the plug and the phone
+// //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 0);
+// ppListOfListListOn = PcapPacketUtils.sortSignatures(ppListOfListListOn);
+// PcapPacketUtils.printSignatures(ppListOfListListOn);
+// //count = 0;
+// /*for (List<List<PcapPacket>> ll : ppListOfListListOn) {
+// PrintUtils.serializeClustersIntoFile("./onSignature" + ++count + ".sig", ll);
+// ppListOfListReadOn.add(PrintUtils.deserializeClustersFromFile("./onSignature" + count + ".sig"));
+// }*/
+// PrintUtils.serializeSignatureIntoFile("./onSignature.sig", ppListOfListListOn);
+// ppListOfListReadOn = PrintUtils.deserializeSignatureFromFile("./onSignature.sig");
+//
+// System.out.println("========================================");
+// System.out.println(" Clustering results for OFF ");
+// System.out.println(" Number of clusters: " + offClusters.size());
+// count = 0;
+// List<List<List<PcapPacket>>> ppListOfListReadOff = new ArrayList<>();
+// List<List<List<PcapPacket>>> ppListOfListListOff = new ArrayList<>();
+// for (Cluster<PcapPacketPair> c : offClusters) {
+// System.out.println(String.format("<<< Cluster #%03d (%06d points) >>>", ++count, c.getPoints().size()));
+// System.out.print(PrintUtils.toSummaryString(c));
+// if(c.getPoints().size() > 45 && c.getPoints().size() < 55) {
+// //if(c.getPoints().size() > 25) {
+// // Print to file
+// List<List<PcapPacket>> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c);
+// ppListOfListListOff.add(ppListOfList);
+// }
+// }
+// // TODO: Merging test
+// ppListOfListListOff = PcapPacketUtils.mergeSignatures(ppListOfListListOff, sortedAllConversation);
+// // TODO: Need to remove sequence 69 296 for Blossom device side since it is not a good signature (overlap)!
+//// PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 3);
+// // TODO: Need to remove sequence number 1 for Nest Thermostat since it is not a good signature!
+// //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 1);
+// // TODO: Need to remove sequence number 0 for Arlo Camera since it is not a good signature!
+//// PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 1);
+// // TODO: Need to remove sequence number 2 for ST plug since it is not a good signature!
+// //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 2);
+// // TODO: Need to remove sequence number 0 for TP-Link plug since it is not a good signature!
+// // TODO: This sequence actually belongs to the local communication between the plug and the phone
+// //PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 0);
+// ppListOfListListOff = PcapPacketUtils.sortSignatures(ppListOfListListOff);
+// PcapPacketUtils.printSignatures(ppListOfListListOff);
+// //count = 0;
+// /*for (List<List<PcapPacket>> ll : ppListOfListListOff) {
+// PrintUtils.serializeClustersIntoFile("./offSignature" + ++count + ".sig", ll);
+// ppListOfListReadOff.add(PrintUtils.deserializeClustersFromFile("./offSignature" + count + ".sig"));
+// }*/
+// PrintUtils.serializeSignatureIntoFile("./offSignature.sig", ppListOfListListOff);
+// ppListOfListReadOff = PrintUtils.deserializeSignatureFromFile("./offSignature.sig");
+// System.out.println("========================================");