From: Hamed Gorjiara <hgorjiar@uci.edu>
Date: Wed, 3 Jul 2019 23:56:02 +0000 (-0700)
Subject: Scripts needed for automation of learning process ... Tested on Sypet benchmark ...
X-Git-Url: http://plrg.eecs.uci.edu/git/?p=satune.git;a=commitdiff_plain;h=38c275a74e1bd8fb93a43293aaed664acc9a8b03

Scripts needed for automation of learning process ... Tested on Sypet benchmark ...
---

diff --git a/deploy-cs.sh b/deploy-cs.sh
index 2f41b32..6eafd06 100755
--- a/deploy-cs.sh
+++ b/deploy-cs.sh
@@ -5,13 +5,14 @@ set -e
 
 BASE=../
 SERVERS="dc-4.calit2.uci.edu dc-5.calit2.uci.edu dc-6.calit2.uci.edu dc-7.calit2.uci.edu dc-8.calit2.uci.edu dc-9.calit2.uci.edu dc-10.calit2.uci.edu dc-11.calit2.uci.edu"
+#SERVERS="dc-1.calit2.uci.edu dc-2.calit2.uci.edu dc-3.calit2.uci.edu"
 REMOTEDIR="/scratch/hamed/"
 INFILE="constraint_compiler/"
 SRC="constraint_compiler/src/"
 SHAREDDIR=~/
 OUTFILE=csolver.tar.gz
 USER=hamed
-
+BIN=${REMOTEDIR}${SRC}/bin
 cd $BASE
 
 rm -f $OUTFILE
@@ -19,5 +20,5 @@ tar -czvf $OUTFILE $INFILE
 
 cp $OUTFILE $SHAREDDIR
 for SERVER in $SERVERS; do
-	ssh $USER@$SERVER "cp $SHAREDDIR$OUTFILE $REMOTEDIR; cd $REMOTEDIR; sudo rm -r $SRC; tar -xzvf $OUTFILE; cd $SRC; make clean; ./setup.sh"
+	ssh $USER@$SERVER "cp $SHAREDDIR$OUTFILE $REMOTEDIR; cd $REMOTEDIR; sudo rm -r $SRC; tar -xzvf $OUTFILE; cd $SRC; make clean; ./setup.sh; find -iname csolver -exec rm '{}' \; -exec ln -s $BIN '{}' \;"
 done
diff --git a/src/Scripts/autotunerparser.py b/src/Scripts/autotunerparser.py
new file mode 100644
index 0000000..618fcd6
--- /dev/null
+++ b/src/Scripts/autotunerparser.py
@@ -0,0 +1,97 @@
+import re
+import argparse
+import sys
+
+
+class AutoTunerArgParser:
+    def __init__(self):
+        self.parser = argparse.ArgumentParser(description='Parsing the output log of the CSolver auto tuner ...')
+        self.parser.add_argument('--file', '-f', metavar='out.log', type=str, nargs=1,help='output log of running the autotuner ...')
+        self.args = self.parser.parse_args()
+        
+    def getFileName(self):
+        return self.args.file[0]
+
+HEADER = ["TESTCASE", "SATTIME", "EXECTIME", "PREPROCESS", "ELEMENTOPT", "ELEMENTOPTSETS", "PROXYVARIABLE", "#SubGraph", "NODEENCODING", "EDGEENCODING", "NAIVEENCODER", "ENCODINGGRAPHOPT"]
+
+configs = {"EXECTIME": "-",
+		"SATTIME":"-",
+		"TESTCASE":"-",
+		"PREPROCESS" : "-",
+		"ELEMENTOPT" : "-",
+		"ELEMENTOPTSETS" : "-",
+		"PROXYVARIABLE" : "-",
+		"#SubGraph" : "-",
+		"NODEENCODING" : "-",
+		"EDGEENCODING" : "-",
+		"NAIVEENCODER" :"-",
+		"ENCODINGGRAPHOPT" : "-"
+		}
+
+REGEXES = {"EXECTIME": "CSOLVER solve time: (.*)",
+		"SATTIME":"SAT Solving time: (.*)",
+		"TESTCASE": "deserializing (.+) ...",
+		"PREPROCESS" : "Param PREPROCESS = (.*)range=\[0,1\]",
+		"ELEMENTOPT" : "Param ELEMENTOPT = (.*)range=\[0,1\]",
+		"ELEMENTOPTSETS" : "Param ELEMENTOPTSETS = (.*)range=\[0,1\]",
+		"PROXYVARIABLE" : "Param PROXYVARIABLE = (.*)range=\[1,5\]",
+		"#SubGraph" : "#SubGraph = (.*)",
+		"NODEENCODING" : "Param NODEENCODING = (.*)range=\[0,3\](.*)",
+		"EDGEENCODING" : "Param EDGEENCODING = (.*)range=\[0,2\](.*)",
+		"NAIVEENCODER" : "Param NAIVEENCODER = (.*)range=\[1,3\](.*)",
+		"ENCODINGGRAPHOPT" : "Param ENCODINGGRAPHOPT = (.*)range=\[0,1\]"
+		}
+def reorderEntry(entry):
+	global HEADER
+	result = []
+	for key in HEADER:
+		result.append(entry[key])
+	return result
+
+
+def printHeader(file):
+	global HEADER
+	mystr=""
+	for key in HEADER:
+		 mystr+=key+","
+	print >>file, mystr
+	
+def printConfig(file, data):
+	print data
+	mystr=""
+	for val  in data:
+		 mystr+=str(val)+","
+	print >> file, mystr
+
+
+def main():
+	global configs
+	argprocess = AutoTunerArgParser()
+	output = open("tuner.csv", "w")
+	printHeader(output)
+	result = []
+	with open(argprocess.getFileName()) as file:
+		for line in file:
+			if line.startswith("Mutating"):
+				result.append(reorderEntry(configs))
+			elif line.startswith("Best tuner"):
+				result.append(reorderEntry(configs))
+			else :
+				for regex in REGEXES:
+					p = re.compile(REGEXES[regex])
+					token = p.search(line)
+					if token is not None:
+						if regex == "TESTCASE":
+							configs[regex] = re.search(REGEXES[regex], line).group(1)
+						else:
+							configs[regex] = re.findall("\d+\.?\d*", line)[0]
+
+	#configs["EXECTIME"] = "BEST TUNE:"
+	result.append(reorderEntry(configs))
+	result.sort(key = lambda entry: entry[0])
+	for entry in result:
+		printConfig(output, entry)
+	print "Done with parsing " + argprocess.getFileName()
+
+if __name__ == "__main__":
+	main()
diff --git a/src/Scripts/learnresultgen.sh b/src/Scripts/learnresultgen.sh
new file mode 100755
index 0000000..d34f090
--- /dev/null
+++ b/src/Scripts/learnresultgen.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+# ./learnresultgen.sh [sypet] [learning set = 1, 2, 3, etc.] [algorithm= 1, 2, 3, 4]
+set -e
+
+if [ "$#" -lt 3 ]; then
+        echo "Illegal number of argument"
+        echo "./learnresultgen.sh [sypet] [learning set = 0, 1, 2, 3, etc.] [algorithm = Known Tuner Types: Random Tuner=1, Comp Tuner=2, Kmeans Tuner=3, Simulated Annealing Tuner=4]"
+        exit 1
+fi
+
+SATUNEDIR=$PWD
+BENCHDIR=$SATUNEDIR/Benchmarks/$1
+BIN=$SATUNEDIR/bin
+
+source $SATUNEDIR/Benchmarks/common.sh
+cd $BENCHDIR
+./learn.sh $2 $3
+cd $BIN
+./run.sh analyzemultituner
+cd $SATUNEDIR
+TUNERS=$(find "$BIN" -name "*.tuner")
+for T in $TUNERS; do
+	TUNER=$(basename $T)
+	echo "Running tuner "$TUNER
+	./Scripts/runbench.sh $1 $TIMEOUT $TUNER &> $BIN/$1"-set"$2"-"$TUNER".log"
+	python ./Scripts/autotunerparser.py -f $BIN/$1"-set"$2"-"$TUNER".log"
+	mv tuner.csv $1"-set"$2"-"$TUNER".csv"
+done
+
+mv ./bin ./"bin-"$1"-set"$2"-alg"$3
diff --git a/src/Scripts/remotelearning.py b/src/Scripts/remotelearning.py
new file mode 100644
index 0000000..dccee9c
--- /dev/null
+++ b/src/Scripts/remotelearning.py
@@ -0,0 +1,117 @@
+import re
+import argparse
+import sys
+from threading import Thread
+import subprocess
+import os
+
+# 1) Deploy on all the servers
+# 2) Based on the benchmark selection, run each learning set on a server
+# 3) After being done with that, it sshould calculate the first best 3
+# 4) Run them indivisually
+# 5) Generate the excel sheet!
+SRCDIR="/scratch/hamed/constraint_compiler/src"
+LOCALSRCDIR="/scratch/satcheck/satproject/constraint_compiler/src"
+class ArgParser:
+    def __init__(self):
+        self.parser = argparse.ArgumentParser(description='Parsing the output log of the CSolver auto tuner ...')
+        self.parser.add_argument('--bench', '-b', metavar='sudoku', type=str, nargs=1,help='Benchmark that you want to learn on')
+        self.args = self.parser.parse_args()
+        
+    def getBenchmarkName(self):
+        return self.args.bench[0]
+       
+def deploy():
+	os.system("cd ../; ./deploy-cs.sh")
+
+def getServerNeeded(benchmark):
+	variable = ""
+	with open("./Benchmarks/" + benchmark + "/learn.sh") as f:
+		line = f.readline()
+		while "declare -a LearningSet=" not in line:
+			line = f.readline()
+		while ")" not in line:
+			variable = variable + line
+			line = f.readline()
+		variable = variable + line
+	return variable.count("\"")/2
+
+def getAvailableServerList(needed):
+	global SRCDIR
+	available = []
+	for i in range(4,12):
+		print ("Checking availability for server " + str(i))
+		HOST="dc-"+ str(i) + ".calit2.uci.edu"
+		COMMAND="cd "+SRCDIR+"; python ./Scripts/serverstatus.py"
+		ssh = subprocess.Popen(["ssh", "%s" % HOST, COMMAND],
+			shell=False,
+			stdout=subprocess.PIPE,
+			stderr=subprocess.PIPE)
+		result = ssh.stdout.readlines()
+		if result == []:
+			error = ssh.stderr.readlines()
+			print >>sys.stderr, "ERROR: %s" % error
+		else:
+			print ("Result of running serverStatus: ")
+			print result
+			if "AVAILABLE\n" in result:
+				available.append(i)
+				if len(available) >= needed:
+					break
+	return available
+
+def startLearningProcess(benchmark, server, learningSet):
+	global SRCDIR
+	HOST="dc-"+ str(server) + ".calit2.uci.edu"
+	ALGORITHM = "2"
+	LOGFILE= benchmark + "-" + str(learningSet) + ".log"
+	print("Running benchmark " + benchmark + "(Set="+ str(learningSet)+") on server")
+	COMMAND=("cd "+SRCDIR+"; ./Scripts/learnresultgen.sh " +
+		benchmark + " " + str(learningSet) + " " + ALGORITHM + " &> " + LOGFILE + "; mv *.csv ~/; echo 'SUCCESS'")
+	print("Calling the following command:\n" + COMMAND)
+	ssh = subprocess.Popen(["ssh", "%s" % HOST, COMMAND],
+		shell=False,
+		stdout=subprocess.PIPE,
+		stderr=subprocess.PIPE)
+	result = ssh.stdout.readlines()
+	if result == []:
+		error = ssh.stderr.readlines()
+		print >>sys.stderr, "ERROR: %s" % error
+	else:
+		print ("Result of running serverStatus: ") 
+		print result
+
+def moveCSVFiles():
+	global LOCALSRCDIR
+	os.system("mv ~/*.csv "+ LOCALSRCDIR)
+
+
+def main():
+	benchmark = ArgParser().getBenchmarkName()
+#	print("Deploying on all the servers ...")
+#	deploy()
+	serverNumber = getServerNeeded(benchmark)
+	print("Learning on " + benchmark + " needs " + str(serverNumber) + " servers.")
+	availableServers = getAvailableServerList(serverNumber)
+	print ("Available Server:" + str(availableServers))
+	if serverNumber > len(availableServers):
+		print("Servers are busy. We don't have enough server available for learning ...")
+		sys.exit(1)
+	try:
+		threads = []
+		for i in range(serverNumber):
+			t = Thread(target=startLearningProcess, args=(benchmark, availableServers[i], i, ))
+			t.start()
+			threads.append(t)
+		
+		for t in threads:
+			t.join()
+		moveCSVFiles()
+	except:
+		print("Exception in creating learning thread ...")
+		sys.exit(1)
+
+
+
+if __name__ == "__main__":
+	main()
diff --git a/src/Scripts/runbench.sh b/src/Scripts/runbench.sh
new file mode 100755
index 0000000..ed66315
--- /dev/null
+++ b/src/Scripts/runbench.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+# run as the following:
+# ./runbench.sh [hexiom] [timeout] [tuner.conf]
+# ./runbench.sh [nqueens] [timeout] [tuner.conf]
+# ./runbench.sh [sudoku-csolver] [timeout] [tuner.conf]
+# ./runbench.sh [killerSudoku] [timeout] [tuner.conf]
+
+if [ "$#" -lt 3 ]; then
+        echo "Illegal number of argument"
+        echo "./runbench.sh [benchmark] [timeout] [tuner.conf]"
+        exit 1
+fi
+
+
+BIN=./bin
+DUMP=$(find . -name "*.dump")
+cd $BIN
+for d in $DUMP; do
+	if [[ $d = *$1* ]]; then
+		echo $d
+		./run.sh tunerrun "."$d $2 "../"$3 out.out
+		echo "Best tuner"
+	fi
+done
diff --git a/src/Scripts/runinterpreter.sh b/src/Scripts/runinterpreter.sh
new file mode 100755
index 0000000..759b957
--- /dev/null
+++ b/src/Scripts/runinterpreter.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+# run as the following:
+# ./runalloy.sh [hexiom] [--alloy]
+# ./runalloy.sh [nqueens] [--alloy]
+# ./runalloy.sh [sudoku-csolver] [--alloy]
+# ./runalloy.sh [killerSudoku] [--alloy]
+
+#./run.sh deserializealloytest ../Benchmarks/sudoku-csolver/4x4.dump --alloy
+#./run.sh java edu.mit.csail.sdg.alloy4whole.ExampleAlloyCompilerNoViz satune.als > solution.log
+
+if [ "$#" -lt 2 ]; then
+        echo "Illegal number of argument"
+        echo "./runinterpreter.sh [benchmark] [--alloy/--z3/--smtrat/--mathsat] [timeout]"
+        exit 1
+fi
+
+
+BIN=./bin
+DUMP=$(find . -name "*.dump")
+cd $BIN
+for d in $DUMP; do
+	if [[ $d = *$1* ]]; then
+		echo $d
+		START=$(date +%s.%N)
+		./run.sh deserializealloytest "."$d $2 $3
+		END=$(date +%s.%N)
+		DIFF=$(echo "$END - $START" | bc)
+		echo "CSOLVER solve time: $DIFF"
+		cat solution.sol
+		echo "Best tuner"
+	fi
+done
diff --git a/src/Scripts/serverstatus.py b/src/Scripts/serverstatus.py
new file mode 100644
index 0000000..3f6ffd4
--- /dev/null
+++ b/src/Scripts/serverstatus.py
@@ -0,0 +1,15 @@
+import psutil
+from time import sleep
+# gives a single float value
+avg = 0.0
+count = 0
+for i in range(350):
+	avg = (psutil.cpu_percent() + avg*count)/(count+1)
+	count = count + 1
+	sleep(0.1)
+
+
+if avg> 15:
+	print "BUSY"
+else:
+	print "AVAILABLE"
diff --git a/src/autotunerparser.py b/src/autotunerparser.py
deleted file mode 100644
index 1632f4b..0000000
--- a/src/autotunerparser.py
+++ /dev/null
@@ -1,83 +0,0 @@
-import re
-import argparse
-import sys
-
-
-class AutoTunerArgParser:
-    def __init__(self):
-        self.parser = argparse.ArgumentParser(description='Parsing the output log of the CSolver auto tuner ...')
-        self.parser.add_argument('--file', '-f', metavar='out.log', type=str, nargs=1,help='output log of running the autotuner ...')
-        self.args = self.parser.parse_args()
-        
-    def getFileName(self):
-        return self.args.file[0]
-       
-configs = {"EXECTIME": "-",
-		"SATTIME":"-",
-		"TESTCASE":"-",
-		"PREPROCESS" : "-",
-		"ELEMENTOPT" : "-",
-		"ELEMENTOPTSETS" : "-",
-		"PROXYVARIABLE" : "-",
-		"#SubGraph" : "-",
-		"NODEENCODING" : "-",
-		"EDGEENCODING" : "-",
-		"NAIVEENCODER" :"-",
-		"ENCODINGGRAPHOPT" : "-"
-		}
-
-REGEXES = {"EXECTIME": "CSOLVER solve time: (.*)",
-		"SATTIME":"SAT Solving time: (.*)",
-		"TESTCASE": "deserializing (.+) ...",
-		"PREPROCESS" : "Param PREPROCESS = (.*)range=\[0,1\]",
-		"ELEMENTOPT" : "Param ELEMENTOPT = (.*)range=\[0,1\]",
-		"ELEMENTOPTSETS" : "Param ELEMENTOPTSETS = (.*)range=\[0,1\]",
-		"PROXYVARIABLE" : "Param PROXYVARIABLE = (.*)range=\[1,5\]",
-		"#SubGraph" : "#SubGraph = (.*)",
-		"NODEENCODING" : "Param NODEENCODING = (.*)range=\[0,3\](.*)",
-		"EDGEENCODING" : "Param EDGEENCODING = (.*)range=\[0,2\](.*)",
-		"NAIVEENCODER" : "Param NAIVEENCODER = (.*)range=\[1,3\](.*)",
-		"ENCODINGGRAPHOPT" : "Param ENCODINGGRAPHOPT = (.*)range=\[0,1\]"
-		}
-
-def printHeader(file):
-	global configs
-	mystr=""
-	for config in configs:
-		 mystr+=str(config)+","
-	print >>file, mystr
-	
-def printConfig(file, data):
-	print data
-	mystr=""
-	for config in data:
-		 mystr+=str(data[config])+","
-	print >> file, mystr
-
-def main():
-	global configs
-	argprocess = AutoTunerArgParser()
-	output = open("tuner.csv", "w")
-	printHeader(output)
-	with open(argprocess.getFileName()) as file:
-		for line in file:
-			if line.startswith("Mutating"):
-				printConfig(output,configs)
-			elif line.startswith("Best tuner"):
-				printConfig(output,configs);
-			else :
-				for regex in REGEXES:
-					p = re.compile(REGEXES[regex])
-					token = p.search(line)
-					if token is not None:
-						if regex == "TESTCASE":
-							configs[regex] = re.search(REGEXES[regex], line).group(1)
-						else:
-							configs[regex] = re.findall("\d+\.?\d*", line)[0]
-
- 	configs["EXECTIME"] = "BEST TUNE:"
-	printConfig(output, configs)
-	print "Done with parsing " + argprocess.getFileName()
-
-if __name__ == "__main__":
-	main()
diff --git a/src/runbench.sh b/src/runbench.sh
deleted file mode 100755
index ed66315..0000000
--- a/src/runbench.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/bin/bash
-# run as the following:
-# ./runbench.sh [hexiom] [timeout] [tuner.conf]
-# ./runbench.sh [nqueens] [timeout] [tuner.conf]
-# ./runbench.sh [sudoku-csolver] [timeout] [tuner.conf]
-# ./runbench.sh [killerSudoku] [timeout] [tuner.conf]
-
-if [ "$#" -lt 3 ]; then
-        echo "Illegal number of argument"
-        echo "./runbench.sh [benchmark] [timeout] [tuner.conf]"
-        exit 1
-fi
-
-
-BIN=./bin
-DUMP=$(find . -name "*.dump")
-cd $BIN
-for d in $DUMP; do
-	if [[ $d = *$1* ]]; then
-		echo $d
-		./run.sh tunerrun "."$d $2 "../"$3 out.out
-		echo "Best tuner"
-	fi
-done
diff --git a/src/runinterpreter.sh b/src/runinterpreter.sh
deleted file mode 100755
index 759b957..0000000
--- a/src/runinterpreter.sh
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/bin/bash
-# run as the following:
-# ./runalloy.sh [hexiom] [--alloy]
-# ./runalloy.sh [nqueens] [--alloy]
-# ./runalloy.sh [sudoku-csolver] [--alloy]
-# ./runalloy.sh [killerSudoku] [--alloy]
-
-#./run.sh deserializealloytest ../Benchmarks/sudoku-csolver/4x4.dump --alloy
-#./run.sh java edu.mit.csail.sdg.alloy4whole.ExampleAlloyCompilerNoViz satune.als > solution.log
-
-if [ "$#" -lt 2 ]; then
-        echo "Illegal number of argument"
-        echo "./runinterpreter.sh [benchmark] [--alloy/--z3/--smtrat/--mathsat] [timeout]"
-        exit 1
-fi
-
-
-BIN=./bin
-DUMP=$(find . -name "*.dump")
-cd $BIN
-for d in $DUMP; do
-	if [[ $d = *$1* ]]; then
-		echo $d
-		START=$(date +%s.%N)
-		./run.sh deserializealloytest "."$d $2 $3
-		END=$(date +%s.%N)
-		DIFF=$(echo "$END - $START" | bc)
-		echo "CSOLVER solve time: $DIFF"
-		cat solution.sol
-		echo "Best tuner"
-	fi
-done