CSAPP2e: Proxy lab 解答

　　这次的Proxy lab 是要求实现一个简单的web 代理。与此相关的章节是网络编程和并发编程，其实之前零零星星的看过一些讲HTTP协议的书，但是对于套接字这些都是一知半解，跟着课堂学完这两章突然柳暗花明，再看一些更详细更深入的书，像《HTTP权威指南》，《计算机网络-自顶向下的方法》就明白的多了。

　　下面就说一下这次lab，共有3个部分，第一部分是实现一个单线程代理，接收客户端请求，连接服务器然后转发。第二部分是实现并发，为每一个请求新建一个进程。第三部分是最有趣的，为每个请求建立独立的进程之后，该怎么共享进程之间的数据，也就是缓存呢？这里用到了书上介绍的写者-读者模型。源代码如下（已经通过了测试）。

  1 /*
  2  * [email protected]
  3  * 2014.12.28 18:24
  4  */
  5
  6 #include "csapp.h"
  7
  8 #define MAX_CACHE_SIZE 1049000
  9 #define MAX_OBJECT_SIZE 102400
 10
 11 /* argument for each thread */
 12 struct arg {
 13     int connfd;            /* client socked */
 14     int turn;            /* the time of current request */
 15 };
 16 /* cache block */
 17 struct cache_block {
 18     char data[MAX_OBJECT_SIZE];        /* store the response head and body */
 19     sem_t mutex;                    /* 模仿书上P673,使用写者-读者模型解决并发冲突 */
 20     sem_t w;
 21     int readcnt;
 22
 23     int turn;                        /* the time of request */
 24     sem_t t_mutex;
 25     sem_t t_w;
 26     int t_readcnt;
 27
 28     char url[300];                    /* the url of request */
 29     sem_t url_mutex;
 30     sem_t url_w;
 31     int url_readcnt;
 32
 33 };
 34 /* total 10 cache block */
 35 struct cache_block cache[10];
 36
 37 /* init the block */
 38 void cache_erase(int index);
 39
 40 /* because of concurrency, all operation on cache use following 4 function */
 41 /* write data, url and turn on cache[index] */
 42 void cache_write(int index, char *url,char *data, int turn);
 43 /* read data on cache[index] */
 44 void cache_data_read(int index, char *dst, int turn);
 45 /* read url on cache[index] */
 46 void cache_url_read(int index,char *dst);
 47 /* read turn on cache[index] */
 48 int cache_turn_read(int index);
 49
 50 /* thread for each request */
 51 void *thread(void *connfdp);
 52
 53 /* parse the request line */
 54 void parse_url(char *url, char *hostname, char *query_path, int *port);
 55
 56 /* connect to server, if failed return -1 */
 57 int connect_server(char *hostname,int port,char *query_path);
 58
 59
 60 int main(int argc,char **argv)
 61 {
 62     Signal(SIGPIPE, SIG_IGN);
 63     struct sockaddr_in clientaddr;
 64     int port,listenfd,clientlen;
 65     int turn=1;
 66     pthread_t tid;
 67     struct arg *p;
 68
 69     /* check port */
 70     if(argc!=2) {
 71         fprintf(stderr, "usage: %s <port>\n", argv[0]);
 72         exit(0);
 73     }
 74
 75     // init 10 cache blocks
 76     int i;
 77     for(i=0;i<10;i++)
 78         cache_erase(i);
 79
 80     port=atoi(argv[1]);
 81     listenfd=Open_listenfd(port);
 82     clientlen=sizeof(clientaddr);
 83
 84     while(1) {
 85         p=(int*)Malloc(sizeof(struct arg));
 86         p->connfd=Accept(listenfd,(SA*)&clientaddr,&clientlen);
 87         p->turn =turn++;
 88         /* create thread */
 89         Pthread_create(&tid,NULL,thread,(void *)p);
 90     }
 91     return 0;
 92 }
 93
 94 /* parse request url */
 95 void parse_url(char *ur, char *hostname, char *query_path, int *port)
 96 {
 97     char url[100];
 98     url[0]=‘\0‘;
 99     strcat(url,ur);
100     hostname[0]=query_path[0]=‘\0‘;
101     char *p=strstr(url,"//");        /* skip "http://" and "https://" */
102     if(p!=NULL) {
103         p=p+2;
104     } else {
105         p=url;
106     }
107     char *q=strstr(p,":");            /* read ":<port>" and "/index.html" */
108     if(q!=NULL) {
109         *q=‘\0‘;
110         sscanf(p,"%s",hostname);
111         sscanf(q+1,"%d%s",port,query_path);
112     } else {
113         q=strstr(p,"/");
114         if(q!=NULL) {
115             *q=‘\0‘;
116             sscanf(p,"%s",hostname);
117             *q=‘/‘;
118             sscanf(q,"%s",query_path);
119         } else {
120             sscanf(p,"%s",hostname);
121         }
122         *port=80;
123     }
124     /* the default path */
125     if(strlen(query_path)<=1)
126         strcpy(query_path,"/index.html");
127
128     return;
129 }
130
131 /* connect to server and return socket fd */
132 int connect_server(char *hostname,int port,char *query_path)
133 {
134     static const char *user_agent = "User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:10.0.3) Gecko/20120305 Firefox/10.0.3\r\n";
135     static const char *accept_str= "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\nAccept-Encoding: gzip, deflate\r\n";
136     static const char *connection = "Connection: close\r\nProxy-Connection: close\r\n";
137
138     char buf[MAXLINE];
139     /* connect to server */
140     int proxy_clientfd;
141     proxy_clientfd=open_clientfd(hostname,port);
142
143     /* if failed return */
144     if(proxy_clientfd<0)
145         return proxy_clientfd;
146
147     /* write request to server */
148     sprintf(buf,"GET %s HTTP/1.0\r\n",query_path);
149     Rio_writen(proxy_clientfd,buf,strlen(buf));
150     sprintf(buf,"Host: %s\r\n",hostname);
151     Rio_writen(proxy_clientfd,buf,strlen(buf));
152     Rio_writen(proxy_clientfd,user_agent,strlen(user_agent));
153     Rio_writen(proxy_clientfd,accept_str,strlen(accept_str));
154     Rio_writen(proxy_clientfd,connection,strlen(connection));
155     Rio_writen(proxy_clientfd,"\r\n",strlen("\r\n"));
156     return proxy_clientfd;
157 }
158
159 /*
160  * if there is a finished cache, read and response.
161  * else connect to server
162  */
163 void *thread(void *p)
164 {
165     Pthread_detach(pthread_self());
166     int connfd=((struct arg*)p)->connfd,turn=((struct arg*)p)->turn;
167     free(p);
168
169     char buf[MAXLINE];
170     char method[MAXLINE],version[MAXLINE],url[MAXLINE];
171     char host[MAXLINE],query[MAXLINE];
172     char url_tmp[300],*data_tmp;
173     rio_t rio;
174     int index,port,content_length;
175     int serverfd;
176
177     /* read request line */
178     rio_readinitb(&rio,connfd);
179     rio_readlineb(&rio,buf,MAXLINE);
180     sscanf(buf,"%s %s %s",method,url,version);
181
182     if(strcasecmp(method,"GET")) {
183         /* ignore */
184         printf("Not GET\r\n");
185         Close(connfd);
186         return NULL;
187     }
188     /* ignore client request */
189     do {
190         rio_readlineb(&rio,buf,MAXLINE-1);
191     }while(strcmp(buf,"\r\n"));
192
193     /* find cache block */
194     for(index=0;index<10;index++) {
195         cache_url_read(index,url_tmp);
196         /* the block‘url is same as current url */
197         if(!strcmp(url,url_tmp))
198             break;
199     }
200
201     data_tmp=(char*)Malloc(MAX_OBJECT_SIZE);
202     data_tmp[0]=‘\0‘;
203
204     if(index <10) { /* if have cached */
205         cache_data_read(index,data_tmp,turn);
206         rio_writen(connfd,data_tmp,strlen(data_tmp));
207         Close(connfd);
208         free(data_tmp);
209         return NULL;
210     }
211
212     /* connect to server */
213     parse_url(url,host,query,&port);
214     if((serverfd=connect_server(host,port,query))<0) {
215         /* connect to server failed, return */
216         free(data_tmp);
217         Close(connfd);
218         return NULL;
219     }
220
221     rio_readinitb(&rio,serverfd);
222     content_length=0;
223     /* read response head line */
224     do {
225         int t=rio_readlineb(&rio,buf,MAXLINE-1);
226         if(t<=0)
227             break;
228         strncat(data_tmp,buf,t);
229         if(strstr(buf,"Content-length")!=NULL)
230             sscanf(buf,"Content-length: %d\r\n",&content_length);
231         rio_writen(connfd,buf,t);
232     }while(strcmp(buf,"\r\n"));
233
234     /* read response body */
235     /* response is small enough to cache */
236     if(content_length+strlen(data_tmp)<MAX_OBJECT_SIZE) {
237         while(content_length>0) {
238             int t= rio_readnb(&rio,buf,(content_length<MAXLINE-1)?content_length:MAXLINE-1);
239             if(t<=0)
240                 continue;
241             content_length-=t;
242             strncat(data_tmp,buf,t);
243             rio_writen(connfd,buf,t);
244         }
245         index=0;
246         int i;
247         /* least-recently-used */
248         for(i=1;i<10;i++) {
249             if(cache_turn_read(i)<cache_turn_read(index)) {
250                 index=i;
251             }
252         }
253         /* cache write */
254         cache_write(index,url,data_tmp,turn);
255     }
256     /* ignore store and write to client */
257     else {
258         while(content_length>0) {
259             int t= rio_readnb(&rio,buf,(content_length<MAXLINE-1)?content_length:MAXLINE-1);
260             if(t<=0)
261                 break;
262             content_length-=t;
263             rio_writen(connfd,buf,t);
264         }
265     }
266     Close(connfd);
267     Close(serverfd);
268     free(data_tmp);
269     return NULL;
270 }
271
272 void cache_erase(int index)
273 {
274     /* init all var */
275     cache[index].turn=0;
276     cache[index].url[0]=‘\0‘;
277     cache[index].data[0]=‘\0‘;
278     Sem_init(&cache[index].t_mutex,0,1);
279     Sem_init(&cache[index].t_w,0,1);
280     cache[index].t_readcnt=0;
281     Sem_init(&cache[index].url_mutex,0,1);
282     Sem_init(&cache[index].url_w,0,1);
283     cache[index].url_readcnt=0;
284     Sem_init(&cache[index].mutex,0,1);
285     Sem_init(&cache[index].w,0,1);
286     cache[index].readcnt=0;
287 }
288
289 void cache_write(int index,char *url, char *data, int turn)
290 {
291     /* semaphore */
292     P(&cache[index].url_w);
293     P(&cache[index].w);
294     P(&cache[index].t_w);
295     /* begin write operation */
296     cache[index].turn=turn;
297     strcpy(cache[index].data,data);
298     strcpy(cache[index].url,url);
299     /* end write operation */
300
301     /* semaphore */
302     V(&cache[index].t_w);
303     V(&cache[index].w);
304     V(&cache[index].url_w);
305     return ;
306 }
307
308 void cache_data_read(int index, char *dst, int turn)
309 {
310     /* semaphore */
311     P(&cache[index].mutex);
312     cache[index].readcnt++;
313     if(cache[index].readcnt==1)
314         P(&cache[index].w);
315     V(&cache[index].mutex);
316     P(&cache[index].t_w);
317
318     /* begin read operation */
319     cache[index].turn=turn;
320     strcpy(dst,cache[index].data);
321     /* end read operation */
322
323     /* semphore */
324     V(&cache[index].t_w);
325     P(&cache[index].mutex);
326     cache[index].readcnt--;
327     if(cache[index].readcnt==0)
328         V(&cache[index].w);
329     V(&cache[index].mutex);
330
331     return;
332 }
333
334 void cache_url_read(int index,char *dst)
335 {
336     /* semaphore */
337     P(&cache[index].url_mutex);
338     cache[index].url_readcnt++;
339     if(cache[index].url_readcnt==1)
340         P(&cache[index].url_w);
341     V(&cache[index].url_mutex);
342
343     /* begin read operation */
344     strcpy(dst,cache[index].url);
345     /* end read operation */
346
347     /* semphore */
348     P(&cache[index].url_mutex);
349     cache[index].url_readcnt--;
350     if(cache[index].url_readcnt==0)
351         V(&cache[index].url_w);
352     V(&cache[index].url_mutex);
353
354     return;
355 }
356
357 int cache_turn_read(int index)
358 {
359     int t;
360     /* semaphore */
361     P(&cache[index].t_mutex);
362     cache[index].t_readcnt++;
363     if(cache[index].t_readcnt==1)
364         P(&cache[index].t_w);
365     V(&cache[index].t_mutex);
366
367     /* begin read operation */
368     t=cache[index].turn;
369     /* end read operation */
370
371     /* semphore */
372     P(&cache[index].t_mutex);
373     cache[index].t_readcnt--;
374     if(cache[index].t_readcnt==0)
375         V(&cache[index].t_w);
376     V(&cache[index].t_mutex);
377
378     return t;
379 }