Scripts needed for automation of learning process ... Tested on Sypet benchmark ...

diff --git a/deploy-cs.sh b/deploy-cs.sh
index 2f41b3271e9c7cbe088b2c8f56019ab52a91c92c..6eafd0612ee4c62a04767b16f96e6f7e33ef418f 100755 (executable)
--- a/deploy-cs.sh
+++ b/deploy-cs.sh
@@ -5,13 +5,14 @@ set -e
 
 BASE=../
 SERVERS="dc-4.calit2.uci.edu dc-5.calit2.uci.edu dc-6.calit2.uci.edu dc-7.calit2.uci.edu dc-8.calit2.uci.edu dc-9.calit2.uci.edu dc-10.calit2.uci.edu dc-11.calit2.uci.edu"
 
 BASE=../
 SERVERS="dc-4.calit2.uci.edu dc-5.calit2.uci.edu dc-6.calit2.uci.edu dc-7.calit2.uci.edu dc-8.calit2.uci.edu dc-9.calit2.uci.edu dc-10.calit2.uci.edu dc-11.calit2.uci.edu"

+#SERVERS="dc-1.calit2.uci.edu dc-2.calit2.uci.edu dc-3.calit2.uci.edu"

 REMOTEDIR="/scratch/hamed/"
 INFILE="constraint_compiler/"
 SRC="constraint_compiler/src/"
 SHAREDDIR=~/
 OUTFILE=csolver.tar.gz
 USER=hamed
 REMOTEDIR="/scratch/hamed/"
 INFILE="constraint_compiler/"
 SRC="constraint_compiler/src/"
 SHAREDDIR=~/
 OUTFILE=csolver.tar.gz
 USER=hamed

-
+BIN=${REMOTEDIR}${SRC}/bin

 cd $BASE
 
 rm -f $OUTFILE
 cd $BASE
 
 rm -f $OUTFILE


@@ -19,5 +20,5 @@ tar -czvf $OUTFILE $INFILE
 
 cp $OUTFILE $SHAREDDIR
 for SERVER in $SERVERS; do
 
 cp $OUTFILE $SHAREDDIR
 for SERVER in $SERVERS; do

-       ssh $USER@$SERVER "cp $SHAREDDIR$OUTFILE $REMOTEDIR; cd $REMOTEDIR; sudo rm -r $SRC; tar -xzvf $OUTFILE; cd $SRC; make clean; ./setup.sh"
+       ssh $USER@$SERVER "cp $SHAREDDIR$OUTFILE $REMOTEDIR; cd $REMOTEDIR; sudo rm -r $SRC; tar -xzvf $OUTFILE; cd $SRC; make clean; ./setup.sh; find -iname csolver -exec rm '{}' \; -exec ln -s $BIN '{}' \;"

 done
 done

diff --git a/src/Scripts/autotunerparser.py b/src/Scripts/autotunerparser.py
new file mode 100644 (file)
index 0000000..618fcd6
--- /dev/null
+++ b/src/Scripts/autotunerparser.py
@@ -0,0 +1,97 @@
+import re
+import argparse
+import sys
+
+
+class AutoTunerArgParser:
+    def __init__(self):
+        self.parser = argparse.ArgumentParser(description='Parsing the output log of the CSolver auto tuner ...')
+        self.parser.add_argument('--file', '-f', metavar='out.log', type=str, nargs=1,help='output log of running the autotuner ...')
+        self.args = self.parser.parse_args()
+        
+    def getFileName(self):
+        return self.args.file[0]
+
+HEADER = ["TESTCASE", "SATTIME", "EXECTIME", "PREPROCESS", "ELEMENTOPT", "ELEMENTOPTSETS", "PROXYVARIABLE", "#SubGraph", "NODEENCODING", "EDGEENCODING", "NAIVEENCODER", "ENCODINGGRAPHOPT"]
+
+configs = {"EXECTIME": "-",
+               "SATTIME":"-",
+               "TESTCASE":"-",
+               "PREPROCESS" : "-",
+               "ELEMENTOPT" : "-",
+               "ELEMENTOPTSETS" : "-",
+               "PROXYVARIABLE" : "-",
+               "#SubGraph" : "-",
+               "NODEENCODING" : "-",
+               "EDGEENCODING" : "-",
+               "NAIVEENCODER" :"-",
+               "ENCODINGGRAPHOPT" : "-"
+               }
+
+REGEXES = {"EXECTIME": "CSOLVER solve time: (.*)",
+               "SATTIME":"SAT Solving time: (.*)",
+               "TESTCASE": "deserializing (.+) ...",
+               "PREPROCESS" : "Param PREPROCESS = (.*)range=\[0,1\]",
+               "ELEMENTOPT" : "Param ELEMENTOPT = (.*)range=\[0,1\]",
+               "ELEMENTOPTSETS" : "Param ELEMENTOPTSETS = (.*)range=\[0,1\]",
+               "PROXYVARIABLE" : "Param PROXYVARIABLE = (.*)range=\[1,5\]",
+               "#SubGraph" : "#SubGraph = (.*)",
+               "NODEENCODING" : "Param NODEENCODING = (.*)range=\[0,3\](.*)",
+               "EDGEENCODING" : "Param EDGEENCODING = (.*)range=\[0,2\](.*)",
+               "NAIVEENCODER" : "Param NAIVEENCODER = (.*)range=\[1,3\](.*)",
+               "ENCODINGGRAPHOPT" : "Param ENCODINGGRAPHOPT = (.*)range=\[0,1\]"
+               }
+def reorderEntry(entry):
+       global HEADER
+       result = []
+       for key in HEADER:
+               result.append(entry[key])
+       return result
+
+
+def printHeader(file):
+       global HEADER
+       mystr=""
+       for key in HEADER:
+                mystr+=key+","
+       print >>file, mystr
+       
+def printConfig(file, data):
+       print data
+       mystr=""
+       for val  in data:
+                mystr+=str(val)+","
+       print >> file, mystr
+
+
+def main():
+       global configs
+       argprocess = AutoTunerArgParser()
+       output = open("tuner.csv", "w")
+       printHeader(output)
+       result = []
+       with open(argprocess.getFileName()) as file:
+               for line in file:
+                       if line.startswith("Mutating"):
+                               result.append(reorderEntry(configs))
+                       elif line.startswith("Best tuner"):
+                               result.append(reorderEntry(configs))
+                       else :
+                               for regex in REGEXES:
+                                       p = re.compile(REGEXES[regex])
+                                       token = p.search(line)
+                                       if token is not None:
+                                               if regex == "TESTCASE":
+                                                       configs[regex] = re.search(REGEXES[regex], line).group(1)
+                                               else:
+                                                       configs[regex] = re.findall("\d+\.?\d*", line)[0]
+
+       #configs["EXECTIME"] = "BEST TUNE:"
+       result.append(reorderEntry(configs))
+       result.sort(key = lambda entry: entry[0])
+       for entry in result:
+               printConfig(output, entry)
+       print "Done with parsing " + argprocess.getFileName()
+
+if __name__ == "__main__":
+       main()

diff --git a/src/Scripts/learnresultgen.sh b/src/Scripts/learnresultgen.sh
new file mode 100755 (executable)
index 0000000..d34f090
--- /dev/null
+++ b/src/Scripts/learnresultgen.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+# ./learnresultgen.sh [sypet] [learning set = 1, 2, 3, etc.] [algorithm= 1, 2, 3, 4]
+set -e
+
+if [ "$#" -lt 3 ]; then
+        echo "Illegal number of argument"
+        echo "./learnresultgen.sh [sypet] [learning set = 0, 1, 2, 3, etc.] [algorithm = Known Tuner Types: Random Tuner=1, Comp Tuner=2, Kmeans Tuner=3, Simulated Annealing Tuner=4]"
+        exit 1
+fi
+
+SATUNEDIR=$PWD
+BENCHDIR=$SATUNEDIR/Benchmarks/$1
+BIN=$SATUNEDIR/bin
+
+source $SATUNEDIR/Benchmarks/common.sh
+cd $BENCHDIR
+./learn.sh $2 $3
+cd $BIN
+./run.sh analyzemultituner
+cd $SATUNEDIR
+TUNERS=$(find "$BIN" -name "*.tuner")
+for T in $TUNERS; do
+       TUNER=$(basename $T)
+       echo "Running tuner "$TUNER
+       ./Scripts/runbench.sh $1 $TIMEOUT $TUNER &> $BIN/$1"-set"$2"-"$TUNER".log"
+       python ./Scripts/autotunerparser.py -f $BIN/$1"-set"$2"-"$TUNER".log"
+       mv tuner.csv $1"-set"$2"-"$TUNER".csv"
+done
+
+mv ./bin ./"bin-"$1"-set"$2"-alg"$3

diff --git a/src/Scripts/remotelearning.py b/src/Scripts/remotelearning.py
new file mode 100644 (file)
index 0000000..dccee9c
--- /dev/null
+++ b/src/Scripts/remotelearning.py
@@ -0,0 +1,117 @@
+import re
+import argparse
+import sys
+from threading import Thread
+import subprocess
+import os
+
+# 1) Deploy on all the servers
+# 2) Based on the benchmark selection, run each learning set on a server
+# 3) After being done with that, it sshould calculate the first best 3
+# 4) Run them indivisually
+# 5) Generate the excel sheet!
+SRCDIR="/scratch/hamed/constraint_compiler/src"
+LOCALSRCDIR="/scratch/satcheck/satproject/constraint_compiler/src"
+class ArgParser:
+    def __init__(self):
+        self.parser = argparse.ArgumentParser(description='Parsing the output log of the CSolver auto tuner ...')
+        self.parser.add_argument('--bench', '-b', metavar='sudoku', type=str, nargs=1,help='Benchmark that you want to learn on')
+        self.args = self.parser.parse_args()
+        
+    def getBenchmarkName(self):
+        return self.args.bench[0]
+       
+def deploy():
+       os.system("cd ../; ./deploy-cs.sh")
+
+def getServerNeeded(benchmark):
+       variable = ""
+       with open("./Benchmarks/" + benchmark + "/learn.sh") as f:
+               line = f.readline()
+               while "declare -a LearningSet=" not in line:
+                       line = f.readline()
+               while ")" not in line:
+                       variable = variable + line
+                       line = f.readline()
+               variable = variable + line
+       return variable.count("\"")/2
+
+def getAvailableServerList(needed):
+       global SRCDIR
+       available = []
+       for i in range(4,12):
+               print ("Checking availability for server " + str(i))
+               HOST="dc-"+ str(i) + ".calit2.uci.edu"
+               COMMAND="cd "+SRCDIR+"; python ./Scripts/serverstatus.py"
+               ssh = subprocess.Popen(["ssh", "%s" % HOST, COMMAND],
+                       shell=False,
+                       stdout=subprocess.PIPE,
+                       stderr=subprocess.PIPE)
+               result = ssh.stdout.readlines()
+               if result == []:
+                       error = ssh.stderr.readlines()
+                       print >>sys.stderr, "ERROR: %s" % error
+               else:
+                       print ("Result of running serverStatus: ")
+                       print result
+                       if "AVAILABLE\n" in result:
+                               available.append(i)
+                               if len(available) >= needed:
+                                       break
+       return available
+
+def startLearningProcess(benchmark, server, learningSet):
+       global SRCDIR
+       HOST="dc-"+ str(server) + ".calit2.uci.edu"
+       ALGORITHM = "2"
+       LOGFILE= benchmark + "-" + str(learningSet) + ".log"
+       print("Running benchmark " + benchmark + "(Set="+ str(learningSet)+") on server")
+       COMMAND=("cd "+SRCDIR+"; ./Scripts/learnresultgen.sh " +
+               benchmark + " " + str(learningSet) + " " + ALGORITHM + " &> " + LOGFILE + "; mv *.csv ~/; echo 'SUCCESS'")
+       print("Calling the following command:\n" + COMMAND)
+       ssh = subprocess.Popen(["ssh", "%s" % HOST, COMMAND],
+               shell=False,
+               stdout=subprocess.PIPE,
+               stderr=subprocess.PIPE)
+       result = ssh.stdout.readlines()
+       if result == []:
+               error = ssh.stderr.readlines()
+               print >>sys.stderr, "ERROR: %s" % error
+       else:
+               print ("Result of running serverStatus: ") 
+               print result
+
+def moveCSVFiles():
+       global LOCALSRCDIR
+       os.system("mv ~/*.csv "+ LOCALSRCDIR)
+
+
+def main():
+       benchmark = ArgParser().getBenchmarkName()
+#      print("Deploying on all the servers ...")
+#      deploy()
+       serverNumber = getServerNeeded(benchmark)
+       print("Learning on " + benchmark + " needs " + str(serverNumber) + " servers.")
+       availableServers = getAvailableServerList(serverNumber)
+       print ("Available Server:" + str(availableServers))
+       if serverNumber > len(availableServers):
+               print("Servers are busy. We don't have enough server available for learning ...")
+               sys.exit(1)
+       try:
+               threads = []
+               for i in range(serverNumber):
+                       t = Thread(target=startLearningProcess, args=(benchmark, availableServers[i], i, ))
+                       t.start()
+                       threads.append(t)
+               
+               for t in threads:
+                       t.join()
+               moveCSVFiles()
+       except:
+               print("Exception in creating learning thread ...")
+               sys.exit(1)
+
+
+
+if __name__ == "__main__":
+       main()

diff --git a/src/Scripts/runbench.sh b/src/Scripts/runbench.sh
new file mode 100755 (executable)
index 0000000..ed66315
--- /dev/null
+++ b/src/Scripts/runbench.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+# run as the following:
+# ./runbench.sh [hexiom] [timeout] [tuner.conf]
+# ./runbench.sh [nqueens] [timeout] [tuner.conf]
+# ./runbench.sh [sudoku-csolver] [timeout] [tuner.conf]
+# ./runbench.sh [killerSudoku] [timeout] [tuner.conf]
+
+if [ "$#" -lt 3 ]; then
+        echo "Illegal number of argument"
+        echo "./runbench.sh [benchmark] [timeout] [tuner.conf]"
+        exit 1
+fi
+
+
+BIN=./bin
+DUMP=$(find . -name "*.dump")
+cd $BIN
+for d in $DUMP; do
+       if [[ $d = *$1* ]]; then
+               echo $d
+               ./run.sh tunerrun "."$d $2 "../"$3 out.out
+               echo "Best tuner"
+       fi
+done

diff --git a/src/Scripts/runinterpreter.sh b/src/Scripts/runinterpreter.sh
new file mode 100755 (executable)
index 0000000..759b957
--- /dev/null
+++ b/src/Scripts/runinterpreter.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+# run as the following:
+# ./runalloy.sh [hexiom] [--alloy]
+# ./runalloy.sh [nqueens] [--alloy]
+# ./runalloy.sh [sudoku-csolver] [--alloy]
+# ./runalloy.sh [killerSudoku] [--alloy]
+
+#./run.sh deserializealloytest ../Benchmarks/sudoku-csolver/4x4.dump --alloy
+#./run.sh java edu.mit.csail.sdg.alloy4whole.ExampleAlloyCompilerNoViz satune.als > solution.log
+
+if [ "$#" -lt 2 ]; then
+        echo "Illegal number of argument"
+        echo "./runinterpreter.sh [benchmark] [--alloy/--z3/--smtrat/--mathsat] [timeout]"
+        exit 1
+fi
+
+
+BIN=./bin
+DUMP=$(find . -name "*.dump")
+cd $BIN
+for d in $DUMP; do
+       if [[ $d = *$1* ]]; then
+               echo $d
+               START=$(date +%s.%N)
+               ./run.sh deserializealloytest "."$d $2 $3
+               END=$(date +%s.%N)
+               DIFF=$(echo "$END - $START" | bc)
+               echo "CSOLVER solve time: $DIFF"
+               cat solution.sol
+               echo "Best tuner"
+       fi
+done

diff --git a/src/Scripts/serverstatus.py b/src/Scripts/serverstatus.py
new file mode 100644 (file)
index 0000000..3f6ffd4
--- /dev/null
+++ b/src/Scripts/serverstatus.py
@@ -0,0 +1,15 @@
+import psutil
+from time import sleep
+# gives a single float value
+avg = 0.0
+count = 0
+for i in range(350):
+       avg = (psutil.cpu_percent() + avg*count)/(count+1)
+       count = count + 1
+       sleep(0.1)
+
+
+if avg> 15:
+       print "BUSY"
+else:
+       print "AVAILABLE"

diff --git a/src/autotunerparser.py b/src/autotunerparser.py
deleted file mode 100644 (file)
index 1632f4b..0000000
--- a/src/autotunerparser.py
+++ /dev/null
@@ -1,83 +0,0 @@
-import re
-import argparse
-import sys
-
-
-class AutoTunerArgParser:
-    def __init__(self):
-        self.parser = argparse.ArgumentParser(description='Parsing the output log of the CSolver auto tuner ...')
-        self.parser.add_argument('--file', '-f', metavar='out.log', type=str, nargs=1,help='output log of running the autotuner ...')
-        self.args = self.parser.parse_args()
-        
-    def getFileName(self):
-        return self.args.file[0]
-       
-configs = {"EXECTIME": "-",
-               "SATTIME":"-",
-               "TESTCASE":"-",
-               "PREPROCESS" : "-",
-               "ELEMENTOPT" : "-",
-               "ELEMENTOPTSETS" : "-",
-               "PROXYVARIABLE" : "-",
-               "#SubGraph" : "-",
-               "NODEENCODING" : "-",
-               "EDGEENCODING" : "-",
-               "NAIVEENCODER" :"-",
-               "ENCODINGGRAPHOPT" : "-"
-               }
-
-REGEXES = {"EXECTIME": "CSOLVER solve time: (.*)",
-               "SATTIME":"SAT Solving time: (.*)",
-               "TESTCASE": "deserializing (.+) ...",
-               "PREPROCESS" : "Param PREPROCESS = (.*)range=\[0,1\]",
-               "ELEMENTOPT" : "Param ELEMENTOPT = (.*)range=\[0,1\]",
-               "ELEMENTOPTSETS" : "Param ELEMENTOPTSETS = (.*)range=\[0,1\]",
-               "PROXYVARIABLE" : "Param PROXYVARIABLE = (.*)range=\[1,5\]",
-               "#SubGraph" : "#SubGraph = (.*)",
-               "NODEENCODING" : "Param NODEENCODING = (.*)range=\[0,3\](.*)",
-               "EDGEENCODING" : "Param EDGEENCODING = (.*)range=\[0,2\](.*)",
-               "NAIVEENCODER" : "Param NAIVEENCODER = (.*)range=\[1,3\](.*)",
-               "ENCODINGGRAPHOPT" : "Param ENCODINGGRAPHOPT = (.*)range=\[0,1\]"
-               }
-
-def printHeader(file):
-       global configs
-       mystr=""
-       for config in configs:
-                mystr+=str(config)+","
-       print >>file, mystr
-       
-def printConfig(file, data):
-       print data
-       mystr=""
-       for config in data:
-                mystr+=str(data[config])+","
-       print >> file, mystr
-
-def main():
-       global configs
-       argprocess = AutoTunerArgParser()
-       output = open("tuner.csv", "w")
-       printHeader(output)
-       with open(argprocess.getFileName()) as file:
-               for line in file:
-                       if line.startswith("Mutating"):
-                               printConfig(output,configs)
-                       elif line.startswith("Best tuner"):
-                               printConfig(output,configs);
-                       else :
-                               for regex in REGEXES:
-                                       p = re.compile(REGEXES[regex])
-                                       token = p.search(line)
-                                       if token is not None:
-                                               if regex == "TESTCASE":
-                                                       configs[regex] = re.search(REGEXES[regex], line).group(1)
-                                               else:
-                                                       configs[regex] = re.findall("\d+\.?\d*", line)[0]
-
-       configs["EXECTIME"] = "BEST TUNE:"
-       printConfig(output, configs)
-       print "Done with parsing " + argprocess.getFileName()
-
-if __name__ == "__main__":
-       main()

diff --git a/src/runbench.sh b/src/runbench.sh
deleted file mode 100755 (executable)
index ed66315..0000000
--- a/src/runbench.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/bin/bash
-# run as the following:
-# ./runbench.sh [hexiom] [timeout] [tuner.conf]
-# ./runbench.sh [nqueens] [timeout] [tuner.conf]
-# ./runbench.sh [sudoku-csolver] [timeout] [tuner.conf]
-# ./runbench.sh [killerSudoku] [timeout] [tuner.conf]
-
-if [ "$#" -lt 3 ]; then
-        echo "Illegal number of argument"
-        echo "./runbench.sh [benchmark] [timeout] [tuner.conf]"
-        exit 1
-fi
-
-
-BIN=./bin
-DUMP=$(find . -name "*.dump")
-cd $BIN
-for d in $DUMP; do
-       if [[ $d = *$1* ]]; then
-               echo $d
-               ./run.sh tunerrun "."$d $2 "../"$3 out.out
-               echo "Best tuner"
-       fi
-done

diff --git a/src/runinterpreter.sh b/src/runinterpreter.sh
deleted file mode 100755 (executable)
index 759b957..0000000
--- a/src/runinterpreter.sh
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/bin/bash
-# run as the following:
-# ./runalloy.sh [hexiom] [--alloy]
-# ./runalloy.sh [nqueens] [--alloy]
-# ./runalloy.sh [sudoku-csolver] [--alloy]
-# ./runalloy.sh [killerSudoku] [--alloy]
-
-#./run.sh deserializealloytest ../Benchmarks/sudoku-csolver/4x4.dump --alloy
-#./run.sh java edu.mit.csail.sdg.alloy4whole.ExampleAlloyCompilerNoViz satune.als > solution.log
-
-if [ "$#" -lt 2 ]; then
-        echo "Illegal number of argument"
-        echo "./runinterpreter.sh [benchmark] [--alloy/--z3/--smtrat/--mathsat] [timeout]"
-        exit 1
-fi
-
-
-BIN=./bin
-DUMP=$(find . -name "*.dump")
-cd $BIN
-for d in $DUMP; do
-       if [[ $d = *$1* ]]; then
-               echo $d
-               START=$(date +%s.%N)
-               ./run.sh deserializealloytest "."$d $2 $3
-               END=$(date +%s.%N)
-               DIFF=$(echo "$END - $START" | bc)
-               echo "CSOLVER solve time: $DIFF"
-               cat solution.sol
-               echo "Best tuner"
-       fi
-done