From: Janus Varmarken Date: Thu, 17 Jan 2019 00:21:38 +0000 (-0800) Subject: Add utility for analyzing the success of a detection run X-Git-Url: http://plrg.eecs.uci.edu/git/?a=commitdiff_plain;h=884d5e68edad0b94de1406ea6f4993606c066bff;p=pingpong.git Add utility for analyzing the success of a detection run --- diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/UserAction.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/UserAction.java index 807ea78..408d66a 100644 --- a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/UserAction.java +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/analysis/UserAction.java @@ -1,6 +1,8 @@ package edu.uci.iotproject.analysis; import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; /** * Models a user's action, such as toggling the smart plug on/off at a given time. @@ -9,6 +11,35 @@ import java.time.Instant; */ public class UserAction { + private static volatile DateTimeFormatter TIMESTAMP_FORMATTER = DateTimeFormatter.ISO_ZONED_DATE_TIME. + withZone(ZoneId.of("America/Los_Angeles")); + + /** + * Sets the {@link DateTimeFormatter} used when outputting a user action as a string and parsing a user action from + * a string. + * @param formatter The formatter to use for outputting and parsing. + */ + public static void setTimestampFormatter(DateTimeFormatter formatter) { + TIMESTAMP_FORMATTER = formatter; + } + + /** + * Instantiates a {@code UserAction} from a string that obeys the format used in {@link UserAction#toString()}. + * @param string The string that represents a {@code UserAction} + * @return A {@code UserAction} resulting from deserializing the string. + */ + public static UserAction fromString(String string) { + String[] parts = string.split("@"); + if (parts.length != 2) { + throw new IllegalArgumentException("Invalid string format"); + } + // If any of these two parses fail, an exception is thrown -- no need to check return values. + UserAction.Type actionType = UserAction.Type.valueOf(parts[0].trim()); + Instant timestamp = TIMESTAMP_FORMATTER.parse(parts[1].trim(), Instant::from); + return new UserAction(actionType, timestamp); + } + + /** * The specific type of action the user performed. */ @@ -72,6 +103,6 @@ public class UserAction { @Override public String toString() { - return String.format("[ %s @ %s ]", mType.name(), mTimestamp.toString()); + return String.format("%s @ %s", mType.name(), TIMESTAMP_FORMATTER.format(mTimestamp)); } } diff --git a/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/evaluation/DetectionResultsAnalyzer.java b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/evaluation/DetectionResultsAnalyzer.java new file mode 100644 index 0000000..3fdacb3 --- /dev/null +++ b/Code/Projects/SmartPlugDetector/src/main/java/edu/uci/iotproject/evaluation/DetectionResultsAnalyzer.java @@ -0,0 +1,86 @@ +package edu.uci.iotproject.evaluation; + +import edu.uci.iotproject.analysis.TriggerTrafficExtractor; +import edu.uci.iotproject.analysis.UserAction; +import edu.uci.iotproject.io.TriggerTimesFileReader; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +/** + * Utility for comparing detected events to logged (actual) events. + * + * @author Janus Varmarken {@literal } + * @author Rahmadi Trimananda {@literal } + */ +public class DetectionResultsAnalyzer { + + public static void main(String[] args) throws IOException { + // -------------------------------------- Parse the input files -------------------------------------- + + String triggerTimesFile = args[0]; + // Read the trigger times. + // The trigger times file does not contain event types as we initially assumed that we would just be alternating + // between ON and OFF. + List triggerTimestamps = new TriggerTimesFileReader().readTriggerTimes(triggerTimesFile, false); + // Now generate user actions based on this alternating ON/OFF pattern. + List triggers = new ArrayList<>(); + for (int i = 0; i < triggerTimestamps.size(); i++) { + // NOTE: assumes triggers alternate between ON and OFF + UserAction.Type actionType = i % 2 == 0 ? UserAction.Type.TOGGLE_ON : UserAction.Type.TOGGLE_OFF; + triggers.add(new UserAction(actionType, triggerTimestamps.get(i))); + } + // Read the detection output file, assuming a format as specified in UserAction.toString() + File detectionOutputFile = new File(args[1]); + List detectedEvents = new ArrayList<>(); + try (BufferedReader br = new BufferedReader(new FileReader(detectionOutputFile))) { + String s; + while ((s = br.readLine()) != null) { + detectedEvents.add(UserAction.fromString(s)); + } + } + + // ----------------- Now ready to compare the detected events with the logged events ----------------- + + // To contain all detected events that could be mapped to a trigger + List truePositives = new ArrayList<>(); + for (UserAction detectedEvent : detectedEvents) { + Optional matchingTrigger = triggers.stream() + .filter(t -> t.getType() == detectedEvent.getType() && + t.getTimestamp().isBefore(detectedEvent.getTimestamp()) && + t.getTimestamp().plusMillis(TriggerTrafficExtractor.INCLUSION_WINDOW_MILLIS). + isAfter(detectedEvent.getTimestamp()) + ).findFirst(); + matchingTrigger.ifPresent(mt -> { + // We've consumed the trigger (matched it with a detected event), so remove it so we don't match with + // another detected event. + triggers.remove(mt); + // The current detected event was a true positive as we could match it with a trigger. + truePositives.add(detectedEvent); + }); + } + // Now the false positives are those elements in detectedEvents that are not in truePositives + List falsePositives = new ArrayList<>(); + falsePositives.addAll(detectedEvents); + falsePositives.removeAll(truePositives); + // Print the results... + System.out.println("---------- False negatives (events that where not detected) ----------"); + for (UserAction missing : triggers) { + System.out.println(missing); + } + System.out.println("Total of " + Integer.toString(triggers.size())); + System.out.println(); + System.out.println("---------- False positives (detected, but no matching trigger) ----------"); + for (UserAction fp : falsePositives) { + System.out.println(fp); + } + System.out.println("Total of " + Integer.toString(falsePositives.size())); + } + +}