From: rtrimana Date: Tue, 26 Mar 2019 21:11:00 +0000 (-0700) Subject: Printing output of signature generation into a log file. X-Git-Url: http://plrg.eecs.uci.edu/git/?p=pingpong.git;a=commitdiff_plain;h=69fdc77c2c145fecc12ebf5329c9cfe2225809c3 Printing output of signature generation into a log file. --- diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/SignatureGenerator.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/SignatureGenerator.java index 164a328..ae8e806 100644 --- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/SignatureGenerator.java +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/SignatureGenerator.java @@ -3,6 +3,7 @@ package edu.uci.iotproject; import static edu.uci.iotproject.analysis.UserAction.Type; import edu.uci.iotproject.analysis.*; +import edu.uci.iotproject.io.PrintWriterUtils; import edu.uci.iotproject.io.TriggerTimesFileReader; import edu.uci.iotproject.trafficreassembly.layer3.Conversation; import edu.uci.iotproject.trafficreassembly.layer3.TcpReassembler; @@ -13,7 +14,7 @@ import org.apache.commons.math3.stat.clustering.DBSCANClusterer; import org.pcap4j.core.*; import org.pcap4j.packet.namednumber.DataLinkType; -import java.io.EOFException; +import java.io.*; import java.net.UnknownHostException; import java.time.Duration; import java.time.Instant; @@ -34,8 +35,21 @@ import java.util.stream.Stream; */ public class SignatureGenerator { + /** + * If set to {@code true}, output written to the results file is also dumped to standard out. + */ + private static boolean DUPLICATE_OUTPUT_TO_STD_OUT = true; + /** + * File name for logging. + */ + private static String LOG_EXTENSION = "_signature-generation.log"; + /** + * Directory for logging. + */ + private static String LOG_DIRECTORY = "./"; - public static void main(String[] args) throws PcapNativeException, NotOpenException, EOFException, TimeoutException, UnknownHostException { + public static void main(String[] args) throws PcapNativeException, NotOpenException, EOFException, + TimeoutException, UnknownHostException, IOException { // ------------------------------------------------------------------------------------------------------------- // ------------ # Code for extracting traffic generated by a device within x seconds of a trigger # ------------ if (args.length < 11) { @@ -71,6 +85,12 @@ public class SignatureGenerator { final double eps = Double.parseDouble(args[8]); final String deletedSequencesOn = args[9]; final String deletedSequencesOff = args[10]; + final String logFile = inputPcapFile + LOG_EXTENSION; + + // Prepare file outputter. + File outputFile = new File(logFile); + outputFile.getParentFile().mkdirs(); + final PrintWriter resultsWriter = new PrintWriter(new FileWriter(outputFile)); // =========================================== TRAFFIC FILTERING ============================================ @@ -107,29 +127,33 @@ public class SignatureGenerator { // Group conversations by hostname. Map> convsByHostname = TcpConversationUtils.groupConversationsByHostname(allConversations, dnsMap); - System.out.println("Grouped conversations by hostname."); + PrintWriterUtils.println("Grouped conversations by hostname.", resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); // For each hostname, count the frequencies of packet lengths exchanged with that hostname. final Map> pktLenFreqsByHostname = new HashMap<>(); convsByHostname.forEach((host, convs) -> pktLenFreqsByHostname.put(host, TcpConversationUtils.countPacketLengthFrequencies(convs))); - System.out.println("Counted frequencies of packet lengths exchanged with each hostname."); + PrintWriterUtils.println("Counted frequencies of packet lengths exchanged with each hostname.", + resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); // For each hostname, count the frequencies of packet sequences (i.e., count how many // conversations exchange a sequence of packets of some specific lengths). final Map> pktSeqFreqsByHostname = new HashMap<>(); convsByHostname.forEach((host, convs) -> pktSeqFreqsByHostname.put(host, TcpConversationUtils.countPacketSequenceFrequencies(convs))); - System.out.println("Counted frequencies of packet sequences exchanged with each hostname."); + PrintWriterUtils.println("Counted frequencies of packet sequences exchanged with each hostname.", + resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); // For each hostname, count frequencies of packet pairs exchanged // with that hostname across all conversations final Map> pktPairFreqsByHostname = TcpConversationUtils.countPacketPairFrequenciesByHostname(allConversations, dnsMap); - System.out.println("Counted frequencies of packet pairs per hostname"); + PrintWriterUtils.println("Counted frequencies of packet pairs per hostname.", + resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); // For each user action, reassemble the set of TCP connections occurring shortly after final Map> userActionToConversations = trafficLabeler.getLabeledReassembledTcpTraffic(); final Map>> userActionsToConvsByHostname = trafficLabeler.getLabeledReassembledTcpTraffic(dnsMap); - System.out.println("Reassembled TCP conversations occurring shortly after each user event"); + PrintWriterUtils.println("Reassembled TCP conversations occurring shortly after each user event.", + resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); /* * NOTE: no need to generate these more complex on/off maps that also contain mappings from hostname and @@ -197,16 +221,20 @@ public class SignatureGenerator { // Sort the conversations as reference List sortedAllConversation = TcpConversationUtils.sortConversationList(allConversations); // Output clusters - System.out.println("========================================"); - System.out.println(" Clustering results for ON "); - System.out.println(" Number of clusters: " + onClusters.size()); + PrintWriterUtils.println("========================================", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println(" Clustering results for ON ", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println(" Number of clusters: " + onClusters.size(), resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); int count = 0; List>> ppListOfListReadOn = new ArrayList<>(); List>> ppListOfListListOn = new ArrayList<>(); List>> corePointRangeSignatureOn = new ArrayList<>(); for (Cluster c : onClusters) { - System.out.println(String.format("<<< Cluster #%02d (%03d points) >>>", ++count, c.getPoints().size())); - System.out.print(PrintUtils.toSummaryString(c)); + PrintWriterUtils.println(String.format("<<< Cluster #%02d (%03d points) >>>", ++count, c.getPoints().size()), + resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println(PrintUtils.toSummaryString(c), resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); if(c.getPoints().size() > lowerBound && c.getPoints().size() < upperBound) { // Print to file List> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c); @@ -215,16 +243,20 @@ public class SignatureGenerator { ppListOfListListOn.add(ppListOfList); } } - System.out.println("========================================"); - System.out.println(" Clustering results for OFF "); - System.out.println(" Number of clusters: " + offClusters.size()); + PrintWriterUtils.println("========================================", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println(" Clustering results for OFF ", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println(" Number of clusters: " + offClusters.size(), resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); count = 0; List>> ppListOfListReadOff = new ArrayList<>(); List>> ppListOfListListOff = new ArrayList<>(); List>> corePointRangeSignatureOff = new ArrayList<>(); for (Cluster c : offClusters) { - System.out.println(String.format("<<< Cluster #%03d (%06d points) >>>", ++count, c.getPoints().size())); - System.out.print(PrintUtils.toSummaryString(c)); + PrintWriterUtils.println(String.format("<<< Cluster #%03d (%06d points) >>>", ++count, c.getPoints().size()), + resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println(PrintUtils.toSummaryString(c), resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); if(c.getPoints().size() > lowerBound && c.getPoints().size() < upperBound) { // Print to file List> ppListOfList = PcapPacketUtils.clusterToListOfPcapPackets(c); @@ -237,9 +269,6 @@ public class SignatureGenerator { // =========================================== SIGNATURE CREATION =========================================== // Concatenate ppListOfListListOn = PcapPacketUtils.concatSequences(ppListOfListListOn, sortedAllConversation); - // TODO: Need to remove sequence number 0 for TP-Link plug since it is not a good signature! - // TODO: This sequence actually belongs to the local communication between the plug and the phone -// PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOn, 0); // Remove sequences in the list that have overlap StringTokenizer stringTokenizerOn = new StringTokenizer(deletedSequencesOn, ","); while(stringTokenizerOn.hasMoreTokens()) { @@ -253,9 +282,6 @@ public class SignatureGenerator { // Concatenate ppListOfListListOff = PcapPacketUtils.concatSequences(ppListOfListListOff, sortedAllConversation); - // TODO: Need to remove sequence number 0 for TP-Link plug since it is not a good signature! - // TODO: This sequence actually belongs to the local communication between the plug and the phone -// PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, 0); // Remove sequences in the list that have overlap StringTokenizer stringTokenizerOff = new StringTokenizer(deletedSequencesOff, ","); while(stringTokenizerOff.hasMoreTokens()) { @@ -266,16 +292,21 @@ public class SignatureGenerator { PcapPacketUtils.removeSequenceFromSignature(ppListOfListListOff, sequenceToDelete); } ppListOfListListOff = PcapPacketUtils.sortSequences(ppListOfListListOff); - // Write the signatures into the screen - System.out.println("========================================"); - System.out.println(" ON Signature "); - System.out.println("========================================"); - PcapPacketUtils.printSignatures(ppListOfListListOn); - System.out.println("========================================"); - System.out.println(" OFF Signature "); - System.out.println("========================================"); - PcapPacketUtils.printSignatures(ppListOfListListOff); + PrintWriterUtils.println("========================================", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println(" ON Signature ", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println("========================================", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); + PcapPacketUtils.printSignatures(ppListOfListListOn, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println("========================================", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println(" OFF Signature ", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println("========================================", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); + PcapPacketUtils.printSignatures(ppListOfListListOff, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); // Printing signatures into files PrintUtils.serializeIntoFile(onSignatureFile, ppListOfListListOn); PrintUtils.serializeIntoFile(offSignatureFile, ppListOfListListOff); @@ -328,9 +359,12 @@ public class SignatureGenerator { Iterator iterLast = lastSignatureTimestamps.iterator(); long duration; long maxDuration = Long.MIN_VALUE; - System.out.println("========================================"); - System.out.println(" Signature Durations "); - System.out.println("========================================"); + PrintWriterUtils.println("========================================", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println(" Signature Durations ", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println("========================================", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); while (iterFirst.hasNext() && iterLast.hasNext()) { Instant firstInst = (Instant) iterFirst.next(); Instant lastInst = (Instant) iterLast.next(); @@ -351,7 +385,7 @@ public class SignatureGenerator { dur = Duration.between(firstInst, lastInst); duration = dur.toMillis(); } - System.out.println(duration); + PrintWriterUtils.println(duration, resultsWriter, DUPLICATE_OUTPUT_TO_STD_OUT); // Update duration if this bigger than the max value and still less than the window inclusion time maxDuration = maxDuration < duration && duration <= TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS ? duration : maxDuration; @@ -360,12 +394,14 @@ public class SignatureGenerator { if (maxDuration == Long.MIN_VALUE) { maxDuration = 0; } - System.out.println("========================================"); - System.out.println("Max signature duration: " + maxDuration); - System.out.println("========================================"); - + PrintWriterUtils.println("========================================", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println("Max signature duration: " + maxDuration, resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); + PrintWriterUtils.println("========================================", resultsWriter, + DUPLICATE_OUTPUT_TO_STD_OUT); + resultsWriter.flush(); + resultsWriter.close(); // ========================================================================================================== } - - -} +} \ No newline at end of file diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/io/PrintWriterUtils.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/io/PrintWriterUtils.java index 167993d..50b6b13 100644 --- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/io/PrintWriterUtils.java +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/io/PrintWriterUtils.java @@ -28,6 +28,20 @@ public final class PrintWriterUtils { writer.println(line); } + /** + * Invoke {@link PrintWriter#println(Object)} passing {@code line} as argument while also printing {@code line} to + * standard output if {@code duplicateToStdOut} is {@code true}. + * @param line The line to be printed. + * @param writer The {@link PrintWriter} that is to print {@code line}. + * @param duplicateToStdOut Set to {@code true} if {@code line} should also be printed in standard output. + */ + public static void print(Object line, PrintWriter writer, boolean duplicateToStdOut) { + if (duplicateToStdOut) { + System.out.print(line); + } + writer.print(line); + } + /** * Make writer (and standard output, if {@code duplicateToStdOut} is {@code true}) print an empty line. * @param writer The writer that {@link PrintWriter#println()} is to be invoked on. diff --git a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java index 2f20945..b453566 100644 --- a/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java +++ b/Code/Projects/PacketLevelSignatureExtractor/src/main/java/edu/uci/iotproject/util/PcapPacketUtils.java @@ -1,5 +1,6 @@ package edu.uci.iotproject.util; +import edu.uci.iotproject.io.PrintWriterUtils; import edu.uci.iotproject.trafficreassembly.layer3.Conversation; import edu.uci.iotproject.analysis.PcapPacketPair; import edu.uci.iotproject.analysis.TcpConversationUtils; @@ -11,6 +12,7 @@ import org.pcap4j.packet.IpV4Packet; import org.pcap4j.packet.TcpPacket; import org.pcap4j.util.MacAddress; +import java.io.PrintWriter; import java.util.*; /** @@ -431,26 +433,32 @@ public final class PcapPacketUtils { * * @param signatures A {@link List} of {@link List} of {@link List} of * {@link PcapPacket} objects that needs to be printed. + * @param resultsWriter PrintWriter object to write into log file. + * @param printToOutput Boolean to decide whether to print out to screen or just log file. */ - public static void printSignatures(List>> signatures) { + public static void printSignatures(List>> signatures, PrintWriter resultsWriter, boolean + printToOutput) { // Iterate over the list of all clusters/sequences int sequenceCounter = 0; for(List> listListPcapPacket : signatures) { // Iterate over every member of a cluster/sequence - System.out.print("====== SEQUENCE " + ++sequenceCounter); - System.out.println(" - " + listListPcapPacket.size() + " MEMBERS ======"); + PrintWriterUtils.print("====== SEQUENCE " + ++sequenceCounter, resultsWriter, printToOutput); + PrintWriterUtils.println(" - " + listListPcapPacket.size() + " MEMBERS ======", resultsWriter, + printToOutput); for(List listPcapPacket : listListPcapPacket) { // Print out packet lengths in a sequence int packetCounter = 0; for(PcapPacket pcapPacket : listPcapPacket) { if(pcapPacket != null) { - System.out.print(pcapPacket.length()); + PrintWriterUtils.print(pcapPacket.length(), resultsWriter, printToOutput); } if(packetCounter < listPcapPacket.size() - 1) { - System.out.print(" "); // Provide space if not last packet + // Provide space if not last packet + PrintWriterUtils.print(" ", resultsWriter, printToOutput); } else { - System.out.println(); // Newline if last packet + // Newline if last packet + PrintWriterUtils.println("", resultsWriter, printToOutput); } packetCounter++; }