WvStreams
debian/libwvstreams-dev/usr/include/wvstreams/wvhttppool.h
1/* -*- Mode: C++ -*-
2 * Worldvisions Weaver Software:
3 * Copyright (C) 1997-2002 Net Integration Technologies, Inc.
4 *
5 * A fast, easy-to-use, parallelizing, pipelining HTTP/1.1 file retriever.
6 *
7 * Just create a WvHttpPool object, add it to your list, and use pool.addurl()
8 * to get a WvStream* that gives you the file you requested.
9 */
10#ifndef __WVHTTPPOOL_H
11#define __WVHTTPPOOL_H
12
13#include "ftpparse.h"
14#include "wvurl.h"
15#include "wvistreamlist.h"
16#include "wvstreamclone.h"
17#include "wvlog.h"
18#include "wvhashtable.h"
19#include "wvbufstream.h"
20#include "wvbuf.h"
21#include "wvcont.h"
22#include "wvtcp.h"
23
24class WvBufUrlStream;
25class WvUrlStream;
26class WvHttpStream;
27
28static const WvString DEFAULT_ANON_PW("weasels@");
29
31{
32 WvString name, value;
33
35 : name(_name), value(_value)
36 {}
37};
38
39
40DeclareWvDict(WvHTTPHeader, WvString, name);
41
42
44{
45public:
46 WvUrl url;
47 WvString headers;
48 WvUrlStream *instream;
49 WvBufUrlStream *outstream;
50 WvStream *putstream;
51
52 bool pipeline_test;
53 bool inuse;
54 bool is_dir;
55 bool create_dirs;
56 WvString method;
57
58 WvUrlRequest(WvStringParm _url, WvStringParm _method, WvStringParm _headers,
59 WvStream *content_source, bool _create_dirs, bool _pipeline_test);
61
62 void done();
63};
64
65DeclareWvList(WvUrlRequest);
66
67
69{
70 WvString linkname;
71 WvUrl url;
72
73 WvUrlLink(WvStringParm _linkname, WvStringParm _url)
74 : linkname(_linkname), url(_url)
75 {}
76};
77DeclareWvList(WvUrlLink);
78
79
81{
82public:
83 WvString url;
84 WvString proto;
85 WvUrlLinkList links; // HTML links or FTP directory listing
86
87 // HTTP stuff...
88 WvString version;
89 int status;
90 WvHTTPHeaderDict headers;
91
92 WvBufUrlStream() : status(0), headers(10)
93 {}
94 virtual ~WvBufUrlStream()
95 {}
96
97public:
98 const char *wstype() const { return "WvBufUrlStream"; }
99};
100
101DeclareWvTable(WvIPPortAddr);
102
103
105{
106public:
107 class Target
108 {
109 public:
110 WvIPPortAddr remaddr;
111 WvString username;
112
113 Target(const WvIPPortAddr &_remaddr, WvStringParm _username)
114 : remaddr(_remaddr), username(_username) {}
115
116 ~Target() {}
117
118 bool operator== (const Target &n2) const
119 { return (username == n2.username && remaddr == n2.remaddr); }
120 };
121 Target target;
122 static int max_requests;
123
124protected:
125 WvLog log;
126 WvUrlRequestList urls, waiting_urls;
127 int request_count;
128 WvUrlRequest *curl; // current url
129 virtual void doneurl() = 0;
130 virtual void request_next() = 0;
131
132public:
133 WvUrlStream(const WvIPPortAddr &_remaddr, WvStringParm _username,
134 WvStringParm logname)
135 : WvStreamClone(new WvTCPConn(_remaddr)), target(_remaddr, _username),
136 log(logname, WvLog::Debug)
137 {
138 request_count = 0;
139 curl = NULL;
140 }
141
142 virtual ~WvUrlStream() {};
143
144 virtual void close() = 0;
145 void addurl(WvUrlRequest *url);
146 void delurl(WvUrlRequest *url);
147 // only implemented in WvHttpStream
148 virtual size_t remaining()
149 { return 0; }
150
151 virtual void execute() = 0;
152
153public:
154 const char *wstype() const { return "WvUrlStream"; }
155};
156
157unsigned WvHash(const WvUrlStream::Target &n);
158
159DeclareWvDict(WvUrlStream, WvUrlStream::Target, target);
160
161
163{
164public:
165 static bool global_enable_pipelining;
166 bool enable_pipelining;
167
168private:
169 int pipeline_test_count;
170 bool ssl;
171 bool sent_url_request; // Have we sent a request to the server yet?
172 WvIPPortAddrTable &pipeline_incompatible;
173 WvString http_response, pipeline_test_response;
174 WvDynBuf putstream_data;
175
176 enum { Unknown, Chunked, ContentLength, Infinity,
177 PostHeadInfinity, PostHeadChunked, PostHeadStream,
178 ChuckInfinity, ChuckChunked, ChuckStream } encoding;
179 size_t bytes_remaining;
180 bool in_chunk_trailer, last_was_pipeline_test, in_doneurl;
181
182 virtual void doneurl();
183 virtual void request_next();
184 void start_pipeline_test(WvUrl *url);
185 WvString request_str(WvUrlRequest *url, bool keep_alive);
186 void send_request(WvUrlRequest *url);
187 void pipelining_is_broken(int why);
188
189public:
190 WvHttpStream(const WvIPPortAddr &_remaddr, WvStringParm _username,
191 bool ssl, WvIPPortAddrTable &_pipeline_incompatible);
192 virtual ~WvHttpStream();
193
194 virtual void close();
195 virtual void pre_select(SelectInfo &si);
196 virtual bool post_select(SelectInfo &si);
197 virtual void execute();
198 virtual size_t remaining()
199 { return bytes_remaining; }
200
201public:
202 const char *wstype() const { return "WvHttpStream"; }
203};
204
205
207{
208 bool logged_in, pasv_acked;
209 WvString password;
210 WvTCPConn *data;
211 time_t last_request_time;
212 bool sure;
213
214 virtual void doneurl();
215 virtual void request_next();
216
217 // Disregard all lines that are of the form "xxx-", meaning that another
218 // line follows. Only the last line is important for us.
219 char *get_important_line();
220
221 // Parse response to "PASV" command and returns a pointer to the address
222 // of the data port (or NULL if it can't parse the response)..
223 // This mucks about with line.
224 WvIPPortAddr *parse_pasv_response(char *line);
225
226 WvString parse_for_links(char *line);
227
228 WvCont cont;
229 void* real_execute(void*);
230
231public:
232 WvFtpStream(const WvIPPortAddr &_remaddr, WvStringParm _username,
233 WvStringParm _password);
234
235 virtual void pre_select(SelectInfo &si);
236 virtual bool post_select(SelectInfo &si);
237 virtual void close();
238 virtual void execute();
239
240public:
241 const char *wstype() const { return "WvFtpStream"; }
242};
243
244
245// FIXME: Rename this to WvUrlPool someday.
247{
248 WvLog log;
249 WvResolver dns;
250 WvUrlStreamDict conns;
251 WvUrlRequestList urls;
252 int num_streams_created;
253 bool sure;
254
255 WvIPPortAddrTable pipeline_incompatible;
256
257public:
258 WvHttpPool();
259 virtual ~WvHttpPool();
260
261 virtual void pre_select(SelectInfo &si);
262 virtual bool post_select(SelectInfo &si);
263 virtual void execute();
264
265 WvBufUrlStream *addurl(WvStringParm _url, WvStringParm _method = "GET",
266 WvStringParm _headers = "",
267 WvStream *content_source = NULL,
268 bool create_dirs = false);
269
270 // For URL uploads. create_dirs should be true if you want all
271 // non-existent directories in _url to be created.
272// WvBufUrlStream *addputurl(WvStringParm _url, WvStringParm _headers,
273// WvStream *s, bool create_dirs = false);
274private:
275 void unconnect(WvUrlStream *s);
276
277public:
278 bool idle() const
279 { return !urls.count(); }
280
281public:
282 const char *wstype() const { return "WvHttpPool"; }
283};
284
285
286#endif // __WVHTTPPOOL_H
WvBufStream stores data written by write(), and returns it later on in read().
WvCont provides "continuations", which are apparently also known as semi-coroutines.
A WvFastString acts exactly like a WvString, but can take (const char *) strings without needing to a...
virtual void close()
Close this stream.
virtual void execute()
The callback() function calls execute(), and then calls the user- specified callback if one is define...
virtual void pre_select(SelectInfo &si)
pre_select() sets up for eventually calling select().
virtual bool post_select(SelectInfo &si)
post_select() is called after select(), and returns true if this object is now ready.
virtual void execute()
The callback() function calls execute(), and then calls the user- specified callback if one is define...
virtual void pre_select(SelectInfo &si)
pre_select() sets up for eventually calling select().
virtual bool post_select(SelectInfo &si)
post_select() is called after select(), and returns true if this object is now ready.
virtual void pre_select(SelectInfo &si)
pre_select() sets up for eventually calling select().
virtual bool post_select(SelectInfo &si)
post_select() is called after select(), and returns true if this object is now ready.
virtual void close()
Close this stream.
virtual void execute()
The callback() function calls execute(), and then calls the user- specified callback if one is define...
An IP+Port address also includes a port number, with the resulting form www.xxx.yyy....
WvStreamList holds a list of WvStream objects – and its select() and callback() functions know how to...
A WvLog stream accepts log messages from applications and forwards them to all registered WvLogRcv's.
ASynchronous DNS resolver functions, so that we can do non-blocking lookups.
WvStreamClone simply forwards all requests to the "cloned" stream.
Unified support for streams, that is, sequences of bytes that may or may not be ready for read/write ...
WvString is an implementation of a simple and efficient printable-string class.
WvTCPConn tries to make all outgoing connections asynchronously (in the background).
virtual void close()=0
Close this stream.
virtual void execute()=0
The callback() function calls execute(), and then calls the user- specified callback if one is define...
the data structure used by pre_select()/post_select() and internally by select().