Robust/src/Benchmarks/Spider/BR/Spider.java

   1 task Startup(StartupObject s {initialstate}) {
   2     String firstmachine=s.parameters[0];
   3     String firstpage=s.parameters[1];
   4     QueryList ql=new QueryList() {initialized};
   5     Query firstquery=new Query(firstmachine, firstpage){};
   6     taskexit(s{!initialstate});
   7 }
   8
   9 task requestQuery(Query q{!requested}) {
  10     String hostname=q.getHostName();
  11     q.makeConnection();
  12     StringBuffer req=new StringBuffer("GET ");
  13     req.append("/");
  14     req.append(q.getPath());
  15     req.append(" HTTP/1.1\r\nHost:");
  16     req.append(q.getHostName());
  17     req.append("\r\n\r\n");
  18     q.write(req.toString().getBytes());
  19     taskexit(q{requested});
  20 }
  21
  22 task readResponse(Query q{requested && ! received && IOPending}) {
  23     //    state 0 - nothing
  24     //    state 1 - \r
  25     //    state 2 - \r\n
  26     //    state 3 - \r\n\r
  27     //    state 4 - \r\n\r\n
  28     if (q.state<4) {
  29         if (q.state==0) {
  30             byte[] b=new byte[1];
  31             int numchars=q.read(b);
  32             if ((numchars==1) && (b[0]=='\r'))
  33                 q.state++;
  34         } else if (q.state==1) {
  35             byte[] b=new byte[1];
  36             int numchars=q.read(b);
  37             if (numchars==1) {
  38                 if (b[0]=='\n')
  39                     q.state++;
  40                 else
  41                     q.state=0;
  42             }
  43         } else if (q.state==2) {
  44             byte[] b=new byte[1];
  45             int numchars=q.read(b);
  46             if (numchars==1) {
  47                 if (b[0]=='\r')
  48                     q.state++;
  49                 else
  50                     q.state=0;
  51             }
  52         } else if (q.state==3) {
  53             byte[] b=new byte[1];
  54             int numchars=q.read(b);
  55             if (numchars==1) {
  56                 if (b[0]=='\n')
  57                     q.state++;
  58                 else
  59                     q.state=0;
  60             }
  61         }
  62     } else {
  63         byte[] buffer=new byte[1024];
  64         int numchars=q.read(buffer);
  65         if (numchars==0) {
  66             q.close();
  67             taskexit(q{received});
  68         } else {
  69             String curr=(new String(buffer)).subString(0,numchars);
  70             q.response.append(curr);
  71         }
  72     }
  73 }
  74
  75 task processPage(Query q{received&&!processed}, QueryList ql{initialized}) {
  76     int index=0;
  77     String href=new String("href=\"");
  78     String searchstr=q.response.toString();
  79     boolean cont=true;
  80     q.outputFile();
  81
  82     while(cont) {
  83         int mindex=searchstr.indexOf(href,index);
  84         if (mindex!=-1) {
  85
  86         int endquote=searchstr.indexOf('"', mindex+href.length());
  87         if (endquote!=-1) {
  88             String match=searchstr.subString(mindex+href.length(), endquote);
  89             String match2=q.makewebcanonical(match);
  90             if (match2!=null&&!ql.checkQuery(match2)) {
  91                 ql.addQuery(match2);
  92                 System.printString(q.getHostName(match));
  93                 System.printString("        ");
  94                 System.printString(q.getPathName(match));
  95                 System.printString("\n");
  96                 Query newq=new Query(q.getHostName(match), q.getPathName(match)){};
  97             }
  98             index=endquote;
  99         } else cont=false;
 100         } else cont=false;
 101     }
 102     taskexit(q{processed});
 103 }