Robust/src/Benchmarks/Spider/dsm/QueryThread.java

   1 public class QueryThread extends Task {
   2         int maxDepth;
   3         int depthCnt;
   4         int maxSearchDepth;
   5         int searchDepthCnt;
   6
   7   public QueryThread(Queue qq, Queue ql, int depth, int searchDepth) {
   8     this.todoList = qq;
   9                 this.doneList = ql;
  10                 this.maxDepth = depth;
  11                 this.maxSearchDepth = searchDepth;
  12                 depthCnt = 1;
  13                 searchDepthCnt = 0;
  14   }
  15
  16   public void execute(Object mywork) {
  17                 Query q = (Query)mywork;
  18                 GlobalString ghostname;
  19                 GlobalString gpath;
  20
  21                 atomic {
  22                         ghostname = q.getHostName();
  23                         gpath = q.getPath();
  24                 }
  25
  26                 String hostname = new String(GlobalString.toLocalCharArray(ghostname));
  27                 String path = new String(GlobalString.toLocalCharArray(gpath));
  28
  29                 System.printString("Processing ");
  30                 System.printString(hostname + "\n");
  31                 System.printString(" ");
  32                 System.printString(path);
  33                 System.printString("\n");
  34
  35                 Socket s = new Socket(hostname, 80);
  36
  37                 requestQuery(hostname, path, s);
  38 //              System.printString("Wait for 5 secs\n");
  39 //              Thread.sleep(2000000);
  40
  41                 readResponse(q, s);
  42 //              System.printString("Wait for 5 secs\n");
  43 //              Thread.sleep(2000000);
  44
  45                 q.outputFile();
  46 //              System.printString("Wait for 5 secs\n");
  47 //              Thread.sleep(2000000);
  48
  49                 processPage(q, (QueryList)doneList);
  50                 s.close();
  51   }
  52
  53         public void requestQuery(String hostname, String path, Socket sock) {
  54     StringBuffer req = new StringBuffer("GET ");
  55     req.append("/");
  56                 req.append(path);
  57     req.append(" HTTP/1.1\r\nHost:");
  58     req.append(hostname);
  59     req.append("\r\n\r\n");
  60                 System.printString("req : " + req + "\n");
  61     sock.write(req.toString().getBytes());
  62   }
  63
  64         public void readResponse(Query q, Socket sock) {
  65         //    state 0 - nothing
  66         //    state 1 - \r
  67         //    state 2 - \r\n
  68         //    state 3 - \r\n\r
  69         //    state 4 - \r\n\r\n
  70     int state=0;
  71     while(true) {
  72       if (state<4) {
  73         if (state==0) {
  74           byte[] b=new byte[1];
  75           int numchars=sock.read(b);
  76           if ((numchars==1)) {
  77             if (b[0]=='\r') {
  78               state++;
  79             }
  80           } else
  81                                                 return;
  82         } else if (state==1) {
  83           byte[] b=new byte[1];
  84           int numchars=sock.read(b);
  85           if (numchars==1) {
  86             if (b[0]=='\n')
  87               state++;
  88             else
  89               state=0;
  90           } else return;
  91         } else if (state==2) {
  92           byte[] b=new byte[1];
  93           int numchars=sock.read(b);
  94           if (numchars==1) {
  95             if (b[0]=='\r')
  96               state++;
  97             else
  98               state=0;
  99           } else return;
 100         } else if (state==3) {
 101           byte[] b=new byte[1];
 102           int numchars=sock.read(b);
 103           if (numchars==1) {
 104             if (b[0]=='\n')
 105               state++;
 106             else
 107               state=0;
 108           } else return;
 109         }
 110       } else {
 111                                 byte[] buffer=new byte[1024];
 112         int numchars=sock.read(buffer);
 113         if (numchars==0)
 114           return;
 115         else {
 116           String curr=(new String(buffer)).subString(0,numchars);
 117                                         q.response.append(curr);
 118         }
 119       }
 120     }
 121   }
 122
 123         public void done(Object obj) {
 124                 doneList.push(obj);
 125 //              System.printString("Size of todoList : " + todoList.size() + "\n");
 126 //              Thread.sleep(5000000);
 127         }
 128
 129   public void processPage(Query q, QueryList doneList) {
 130     int index = 0;
 131         String href = new String("href=\"");
 132         String searchstr = q.response.toLocalString();
 133         boolean cont = true;
 134
 135                 while(cont && (searchDepthCnt < maxSearchDepth)) {
 136                         int mindex = searchstr.indexOf(href,index);
 137                         if (mindex != -1) {
 138                                 int endquote = searchstr.indexOf('"', mindex+href.length());
 139                 if (endquote != -1) {
 140                       String match = searchstr.subString(mindex+href.length(), endquote);
 141                                         GlobalString gmatch;
 142                                         GlobalString gmatch2;
 143
 144                                         atomic {
 145                                                 gmatch = global new GlobalString(match);
 146                                                 gmatch2 = q.makewebcanonical(gmatch);
 147                                         }
 148                       if (gmatch2 != null && !doneList.checkQuery(gmatch2)) {
 149 //                                              doneList.push(gmatch2);
 150                                                 done(gmatch2);
 151                                                 if (depthCnt < maxDepth) {
 152                                                         Query newq;
 153                                                         System.printString("Depth : " + depthCnt + "\n");
 154                                                         atomic {
 155                                                                 newq = global new Query(q.getHostName(gmatch), q.getPathName(gmatch));
 156                                                                 todoList.push(newq);
 157                                                                 System.printString("Size of todoList : " + todoList.size() + "\n");
 158                                                                 searchDepthCnt++;
 159                                                         }
 160                                                 }
 161                                         }
 162                       index = endquote;
 163         } else cont = false;
 164       } else cont = false;
 165     }
 166                 depthCnt++;
 167                 searchDepthCnt = 0;
 168   }
 169 }