[w3m-dev 02928] RFC2617: HTTP Digest authentication

* NEWS: RFC2617: HTTP Digest authentication
* config.h.dist (USE_DIGEST_AUTH): added
* configure (use_digest_auth): added, enabled when openssl found
				(because it used openssl/md5.h)
* file.c (auth_param): added
* file.c (http_auth): added
* file.c (extract_auth_val): added
* file.c (qstr_unquote): added
* file.c (extract_auth_param): added
* file.c (get_auth_param): added
* file.c (AuthBasicCred): added
* file.c (digest_hex): added
* file.c (AuthDigestCred): added
* file.c (basic_auth_param): added
* file.c (digest_auth_param): added
* file.c (www_auth): added
* file.c (findAuthentication): added
* file.c (getAuthCookie): rewrite, use http_auth
* file.c (loadGeneralFile): HRequest hr
* file.c (loadGeneralFile): use findAuthentication, new getAuthCookie
* proto.h (HTTPrequestMethod): added
* proto.h (HTTPrequestURI): added
* proto.h (openURL): add `hr' arg
* url.c (HTTPrequestMethod): added
* url.c (HTTPrequestURI): added
* url.c (HTTPrequest): use HTTPrequestMethod and HTTPrequestURI
* url.c (openURL): add `hr' arg
From: Fumitoshi UKAI  <ukai@debian.or.jp>

7 files changed, 437 insertions, 75 deletions

diff --git a/ChangeLog b/ChangeLog
index 7d4fa0b..8617732 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,34 @@
+2002-01-30  Fumitoshi UKAI  <ukai@debian.or.jp>
+
+	* [w3m-dev 02928] RFC2617: HTTP Digest authentication
+	* NEWS: RFC2617: HTTP Digest authentication
+	* config.h.dist (USE_DIGEST_AUTH): added
+	* configure (use_digest_auth): added, enabled when openssl found
+				(because it used openssl/md5.h)
+	* file.c (auth_param): added
+	* file.c (http_auth): added
+	* file.c (extract_auth_val): added
+	* file.c (qstr_unquote): added
+	* file.c (extract_auth_param): added
+	* file.c (get_auth_param): added
+	* file.c (AuthBasicCred): added
+	* file.c (digest_hex): added
+	* file.c (AuthDigestCred): added
+	* file.c (basic_auth_param): added
+	* file.c (digest_auth_param): added
+	* file.c (www_auth): added
+	* file.c (findAuthentication): added
+	* file.c (getAuthCookie): rewrite, use http_auth
+	* file.c (loadGeneralFile): HRequest hr
+	* file.c (loadGeneralFile): use findAuthentication, new getAuthCookie
+	* proto.h (HTTPrequestMethod): added
+	* proto.h (HTTPrequestURI): added
+	* proto.h (openURL): add `hr' arg
+	* url.c (HTTPrequestMethod): added
+	* url.c (HTTPrequestURI): added
+	* url.c (HTTPrequest): use HTTPrequestMethod and HTTPrequestURI
+	* url.c (openURL): add `hr' arg
+
 2002-01-30  SASAKI Takeshi <sasaki@ct.sakura.ne.jp>
 
 	* [w3m-dev 02927] mailto: POST method with w3mmail.cgi
@@ -2362,4 +2393,4 @@
 	* release-0-2-1
 	* import w3m-0.2.1
 
-$Id: ChangeLog,v 1.272 2002/01/29 17:16:35 ukai Exp $
+$Id: ChangeLog,v 1.273 2002/01/29 19:08:49 ukai Exp $
diff --git a/NEWS b/NEWS
index 0706961..59c2916 100644
--- a/NEWS
+++ b/NEWS
@@ -1,5 +1,6 @@
 w3m 0.3?
 
+* RFC2617: HTTP Digest authentication
 * rc: default_url=0(empty) 1(current URL) 2(link URL)
 * GOTO_RELATIVE (M-u)
 * highlight for incremental search
diff --git a/config.h.dist b/config.h.dist
index 2104f07..8c88b6d 100644
--- a/config.h.dist
+++ b/config.h.dist
@@ -113,6 +113,7 @@ MODEL=Linux.i686-monster-ja
 #undef USE_SYSMOUSE
 #define USE_MENU
 #define USE_COOKIE
+#define USE_DIGEST_AUTH
 #define USE_SSL
 #undef USE_SSL_VERIFY
 #undef FTPPASS_HOSTNAMEGEN
diff --git a/configure b/configure
index 0cdf96a..b287c04 100755
--- a/configure
+++ b/configure
@@ -1,5 +1,5 @@
 #!/bin/sh
-# $Id: configure,v 1.57 2002/01/29 16:23:37 ukai Exp $
+# $Id: configure,v 1.58 2002/01/29 19:08:49 ukai Exp $
 #	Configuration.
 #
 
@@ -685,9 +685,12 @@ if [ "$use_ssl" = y ]; then
 fi
 if [ "$use_ssl" = y ]; then
   ask_param "SSL verification support (SSL library >= version 0.8)" use_ssl_verify n
+  ask_param "Digest Auth support" use_digest_auth y
 else
   use_ssl_verify=n
   def_param "use_ssl_verify" n
+  use_digest_auth=n
+  def_param "use_digest_auth" n
 fi
 
 if [ "$use_color" = y ]; then
@@ -2045,6 +2048,7 @@ $def_use_gpm
 $def_use_sysmouse
 $def_use_menu
 $def_use_cookie
+$def_use_digest_auth
 $def_use_ssl
 $def_use_ssl_verify
 $def_ftppass_hostnamegen
diff --git a/file.c b/file.c
index 265f1c3..b83891a 100644
--- a/file.c
+++ b/file.c
@@ -1,4 +1,4 @@
-/* $Id: file.c,v 1.48 2002/01/29 17:16:35 ukai Exp $ */
+/* $Id: file.c,v 1.49 2002/01/29 19:08:50 ukai Exp $ */
 #include "fm.h"
 #include <sys/types.h>
 #include "myctype.h"
@@ -841,36 +841,337 @@ checkContentType(Buffer *buf)
     return r->ptr;
 }
 
+struct auth_param {
+    char *name;
+    Str val;
+};
+
+struct http_auth {
+    int pri;
+    char *scheme;
+    struct auth_param *param;
+    Str (*cred) (struct http_auth * ha, Str uname, Str pw, ParsedURL *pu,
+		 HRequest *hr);
+};
+
+#define TOKEN_PAT	"[^][()<>@,;:\\\"/?={} \t\001-\037\177]*"
+
 static Str
-extractRealm(char *q)
+extract_auth_val(char **q)
 {
-    Str p = Strnew();
-    char c;
+    unsigned char *qq = *(unsigned char **)q;
+    int quoted = 0;
+    Str val = Strnew();
+
+    SKIP_BLANKS(qq);
+    if (*qq == '"') {
+	quoted = TRUE;
+	Strcat_char(val, *qq++);
+    }
+    while (*qq != '\0') {
+	if (quoted && *qq == '"') {
+	    Strcat_char(val, *qq++);
+	    break;
+	}
+	if (!quoted) {
+	    switch (*qq) {
+	    case '(':
+	    case ')':
+	    case '<':
+	    case '>':
+	    case '@':
+	    case ',':
+	    case ';':
+	    case ':':
+	    case '\\':
+	    case '"':
+	    case '/':
+	    case '?':
+	    case '=':
+	    case ' ':
+	    case '\t':
+		qq++;
+		goto end_token;
+	    default:
+		if (*qq <= 037 || *qq == 177) {
+		    qq++;
+		    goto end_token;
+		}
+	    }
+	}
+	else if (quoted && *qq == '\\')
+	    Strcat_char(val, *qq++);
+	Strcat_char(val, *qq++);
+    }
+  end_token:
+    if (*qq != '\0') {
+	SKIP_BLANKS(qq);
+	if (*qq == ',')
+	    qq++;
+    }
+    *q = qq;
+    return val;
+}
 
-    SKIP_BLANKS(q);
-    if (strncasecmp(q, "Basic ", 6) != 0) {
-	/* complicated authentication... not implemented */
-	return NULL;
+static Str
+qstr_unquote(Str s)
+{
+    char *p = s->ptr;
+    if (*p == '"') {
+	Str tmp = Strnew();
+	for (p++; *p != '\0'; p++) {
+	    if (*p == '\\')
+		p++;
+	    Strcat_char(tmp, *p);
+	}
+	if (Strlastchar(tmp) == '"')
+	    Strshrink(tmp, 1);
+	return tmp;
     }
-    q += 6;
-    SKIP_BLANKS(q);
-    if (strncasecmp(q, "realm=", 6) != 0) {
-	/* no realm attribute... get confused */
-	return NULL;
+    else
+	return s;
+}
+
+static char *
+extract_auth_param(char *q, struct auth_param *auth)
+{
+    struct auth_param *ap;
+    char *q0;
+    Regex re_token;
+
+    newRegex(TOKEN_PAT, FALSE, &re_token, NULL);
+
+    for (ap = auth; ap->name != NULL; ap++) {
+	ap->val = NULL;
+    }
+
+    while (*q != '\0') {
+	SKIP_BLANKS(q);
+	for (ap = auth; ap->name != NULL; ap++) {
+	    if (strncasecmp(q, ap->name, strlen(ap->name)) == 0) {
+		q += strlen(ap->name);
+		SKIP_BLANKS(q);
+		if (*q != '=')
+		    return q;
+		q++;
+		ap->val = extract_auth_val(&q);
+		break;
+	    }
+	}
+	if (ap->name == NULL) {
+	    /* skip unknown param */
+	    if (RegexMatch(&re_token, q, -1, TRUE) == 0)
+		return q;
+	    MatchedPosition(&re_token, &q0, &q);
+	    SKIP_BLANKS(q);
+	    if (*q != '=')
+		return q;
+	    q++;
+	    extract_auth_val(&q);
+	}
+    }
+    return q;
+}
+
+static Str
+get_auth_param(struct auth_param *auth, char *name)
+{
+    struct auth_param *ap;
+    for (ap = auth; ap->name != NULL; ap++) {
+	if (strcasecmp(name, ap->name) == 0)
+	    return ap->val;
+    }
+    return NULL;
+}
+
+static Str
+AuthBasicCred(struct http_auth *ha, Str uname, Str pw, ParsedURL *pu,
+	      HRequest *hr)
+{
+    Str s = Strdup(uname);
+    Strcat_char(s, ':');
+    Strcat(s, pw);
+    return encodeB(s->ptr);
+}
+
+#ifdef USE_DIGEST_AUTH
+#include <openssl/md5.h>
+
+/* RFC2617: 3.2.2 The Authorization Request Header
+ * 
+ * credentials      = "Digest" digest-response
+ * digest-response  = 1#( username | realm | nonce | digest-uri
+ *                    | response | [ algorithm ] | [cnonce] |
+ *                     [opaque] | [message-qop] |
+ *                         [nonce-count]  | [auth-param] )
+ *
+ * username         = "username" "=" username-value
+ * username-value   = quoted-string
+ * digest-uri       = "uri" "=" digest-uri-value
+ * digest-uri-value = request-uri   ; As specified by HTTP/1.1
+ * message-qop      = "qop" "=" qop-value
+ * cnonce           = "cnonce" "=" cnonce-value
+ * cnonce-value     = nonce-value
+ * nonce-count      = "nc" "=" nc-value
+ * nc-value         = 8LHEX
+ * response         = "response" "=" request-digest
+ * request-digest = <"> 32LHEX <">
+ * LHEX             =  "0" | "1" | "2" | "3" |
+ *                     "4" | "5" | "6" | "7" |
+ *                     "8" | "9" | "a" | "b" |
+ *                     "c" | "d" | "e" | "f"
+ */
+
+static Str
+digest_hex(char *p)
+{
+    char *h = "0123456789abcdef";
+    Str tmp = Strnew_size(MD5_DIGEST_LENGTH * 2 + 1);
+    int i;
+    for (i = 0; i < MD5_DIGEST_LENGTH; i++, p++) {
+	Strcat_char(tmp, h[(*p >> 4) & 0x0f]);
+	Strcat_char(tmp, h[*p & 0x0f]);
+    }
+    return tmp;
+}
+
+static Str
+AuthDigestCred(struct http_auth *ha, Str uname, Str pw, ParsedURL *pu,
+	       HRequest *hr)
+{
+    Str tmp, a1buf, a2buf, rd, s;
+    char md5[MD5_DIGEST_LENGTH + 1];
+    Str uri = HTTPrequestURI(pu, hr);
+    /*
+     * A1BUF = H(user ":" realm ":" password)
+     * A2BUF = H(method ":" path)
+     * RESPONSE_DIGEST = H(A1BUF ":" nonce ":" A2BUF)
+     */
+
+    tmp = Strnew_m_charp(uname->ptr, ":",
+			 qstr_unquote(get_auth_param(ha->param, "realm"))->ptr,
+			 ":", pw->ptr, NULL);
+    MD5(tmp->ptr, strlen(tmp->ptr), md5);
+    a1buf = digest_hex(md5);
+
+    tmp = Strnew_m_charp(HTTPrequestMethod(hr)->ptr, ":", uri->ptr, NULL);
+    MD5(tmp->ptr, strlen(tmp->ptr), md5);
+    a2buf = digest_hex(md5);
+
+    tmp = Strnew_m_charp(a1buf->ptr, ":",
+			 qstr_unquote(get_auth_param(ha->param, "nonce"))->ptr,
+			 ":", a2buf->ptr, NULL);
+    MD5(tmp->ptr, strlen(tmp->ptr), md5);
+    rd = digest_hex(md5);
+
+    /*
+     * username=<uname>,
+     * realm=<realm>,
+     * nonce=<nonce>,
+     * uri=<uri>,
+     * response=<RESPONSE_DIGEST>,
+     * [opaque=<opaque>]
+     */
+
+    tmp = Strnew_m_charp("username=\"", uname->ptr, "\"", NULL);
+    Strcat_m_charp(tmp, ", realm=",
+		   get_auth_param(ha->param, "realm")->ptr, NULL);
+    Strcat_m_charp(tmp, ", nonce=",
+		   get_auth_param(ha->param, "nonce")->ptr, NULL);
+    Strcat_m_charp(tmp, ", uri=\"", uri->ptr, "\"", NULL);
+    Strcat_m_charp(tmp, ", response=\"", rd->ptr, "\"", NULL);
+
+    if ((s = get_auth_param(ha->param, "opaque")) != NULL)
+	Strcat_m_charp(tmp, ", opaque=", s->ptr, NULL);
+
+    return tmp;
+}
+#endif
+
+/* *INDENT-OFF* */
+struct auth_param basic_auth_param[] = {
+    {"realm", NULL},
+    {NULL, NULL}
+};
+
+#ifdef USE_DIGEST_AUTH
+/* RFC2617: 3.2.1 The WWW-Authenticate Response Header
+ * challenge        =  "Digest" digest-challenge
+ * 
+ * digest-challenge  = 1#( realm | [ domain ] | nonce |
+ *                       [ opaque ] |[ stale ] | [ algorithm ] |
+ *                        [ qop-options ] | [auth-param] )
+ *
+ * domain            = "domain" "=" <"> URI ( 1*SP URI ) <">
+ * URI               = absoluteURI | abs_path
+ * nonce             = "nonce" "=" nonce-value
+ * nonce-value       = quoted-string
+ * opaque            = "opaque" "=" quoted-string
+ * stale             = "stale" "=" ( "true" | "false" )
+ * algorithm         = "algorithm" "=" ( "MD5" | "MD5-sess" |
+ *                        token )
+ * qop-options       = "qop" "=" <"> 1#qop-value <">
+ * qop-value         = "auth" | "auth-int" | token
+ */
+struct auth_param digest_auth_param[] = {
+    {"realm", NULL},
+    {"domain", NULL},
+    {"nonce", NULL},
+    {"opaque", NULL},
+    {"stale", NULL},
+    {"algorithm", NULL},
+    {"qop", NULL},
+    {NULL, NULL}
+};
+#endif
+/* for RFC2617: HTTP Authentication */
+struct http_auth www_auth[] = {
+    { 1, "Basic ", basic_auth_param, AuthBasicCred },
+#ifdef USE_DIGEST_AUTH
+    { 10, "Digest ", digest_auth_param, AuthDigestCred },
+#endif
+    { 0, NULL, NULL, NULL,}
+};
+/* *INDENT-ON* */
+
+static struct http_auth *
+findAuthentication(struct http_auth *hauth, Buffer *buf, char *auth_field)
+{
+    struct http_auth *ha;
+    int len = strlen(auth_field);
+    TextListItem *i;
+    char *p;
+    Regex re_token;
+
+    newRegex(TOKEN_PAT, FALSE, &re_token, NULL);
+
+    bzero(hauth, sizeof(struct http_auth));
+    for (i = buf->document_header->first; i != NULL; i = i->next) {
+	if (strncasecmp(i->ptr, auth_field, len) == 0) {
+	    for (p = i->ptr + len; p != NULL && *p != '\0';) {
+		SKIP_BLANKS(p);
+		for (ha = &www_auth[0]; ha->scheme != NULL; ha++) {
+		    if (strncmp(p, ha->scheme, strlen(ha->scheme)) == 0) {
+			if (hauth->pri < ha->pri) {
+			    *hauth = *ha;
+			    p += strlen(ha->scheme);
+			    SKIP_BLANKS(p);
+			    p = extract_auth_param(p, hauth->param);
+			    break;
+			}
+			else
+			    p += strlen(ha->scheme);
+		    }
+		}
+	    }
+	}
     }
-    q += 6;
-    SKIP_BLANKS(q);
-    c = '\0';
-    if (*q == '"')
-	c = *q++;
-    while (*q != '\0' && *q != c)
-	Strcat_char(p, *q++);
-    return p;
+    return hauth->scheme ? hauth : NULL;
 }
 
 static Str
-getAuthCookie(char *realm, char *auth_header, TextList *extra_header,
-	      ParsedURL *pu)
+getAuthCookie(struct http_auth *hauth, char *auth_header,
+	      TextList *extra_header, ParsedURL *pu, HRequest *hr)
 {
     Str ss;
     Str uname, pwd;
@@ -878,6 +1179,7 @@ getAuthCookie(char *realm, char *auth_header, TextList *extra_header,
     TextListItem *i, **i0;
     int a_found;
     int auth_header_len = strlen(auth_header);
+    char *realm = qstr_unquote(get_auth_param(hauth->param, "realm"))->ptr;
 
     a_found = FALSE;
     for (i0 = &(extra_header->first), i = *i0; i != NULL;
@@ -888,8 +1190,9 @@ getAuthCookie(char *realm, char *auth_header, TextList *extra_header,
 	}
     }
     if (a_found) {
-	/* This means that *-Authenticate: header is received after *
-	 * Authorization: header is sent to the server. */
+	/* This means that *-Authenticate: header is received after
+	 * Authorization: header is sent to the server. 
+	 */
 	if (fmInitialized) {
 	    message("Wrong username or password", 0, 0);
 	    refresh();
@@ -951,18 +1254,17 @@ getAuthCookie(char *realm, char *auth_header, TextList *extra_header,
 				       "Password: "));
 #endif
 	}
-	Strcat_char(uname, ':');
-	Strcat(uname, pwd);
-	ss = encodeB(uname->ptr);
+	ss = hauth->cred(hauth, uname, pwd, pu, hr);
     }
     tmp = Strnew_charp(auth_header);
-    Strcat_charp(tmp, " Basic ");
+    Strcat_m_charp(tmp, " ", hauth->scheme, NULL);
     Strcat(tmp, ss);
     Strcat_charp(tmp, "\r\n");
     pushText(extra_header, tmp->ptr);
     return ss;
 }
 
+
 static int
 same_url_p(ParsedURL *pu1, ParsedURL *pu2)
 {
@@ -997,6 +1299,7 @@ loadGeneralFile(char *path, ParsedURL *volatile current, char *referer,
     unsigned char status = HTST_NORMAL;
     URLOption url_option;
     Str tmp;
+    HRequest hr;
 
     tpath = path;
     prevtrap = NULL;
@@ -1010,7 +1313,7 @@ loadGeneralFile(char *path, ParsedURL *volatile current, char *referer,
     url_option.referer = referer;
     url_option.flag = flag;
     f = openURL(tpath, &pu, current, &url_option, request, extra_header, of,
-		&status);
+		&hr, &status);
     of = NULL;
 #ifdef JP_CHARSET
     content_charset = '\0';
@@ -1193,10 +1496,11 @@ loadGeneralFile(char *path, ParsedURL *volatile current, char *referer,
 	if ((p = checkHeader(t_buf, "WWW-Authenticate:")) != NULL &&
 	    http_response_code == 401) {
 	    /* Authentication needed */
-	    realm = extractRealm(p);
-	    if (realm != NULL) {
-		ss = getAuthCookie(realm->ptr, "Authorization:", extra_header,
-				   &pu);
+	    struct http_auth hauth;
+	    if (findAuthentication(&hauth, t_buf, "WWW-Authenticate:") != NULL
+		&& (realm = get_auth_param(hauth.param, "realm")) != NULL) {
+		ss = getAuthCookie(&hauth, "Authorization:", extra_header,
+				   &pu, &hr);
 		if (ss == NULL) {
 		    /* abort */
 		    UFclose(&f);
@@ -1212,11 +1516,12 @@ loadGeneralFile(char *path, ParsedURL *volatile current, char *referer,
 	if ((p = checkHeader(t_buf, "Proxy-Authenticate:")) != NULL &&
 	    http_response_code == 407) {
 	    /* Authentication needed */
-	    realm = extractRealm(p);
-	    if (realm != NULL) {
-		ss = getAuthCookie(realm->ptr,
-				   "Proxy-Authorization:",
-				   extra_header, &HTTP_proxy_parsed);
+	    struct http_auth hauth;
+	    if (findAuthentication(&hauth, t_buf, "Proxy-Authenticate:")
+		!= NULL
+		&& (realm = get_auth_param(hauth.param, "realm")) != NULL) {
+		ss = getAuthCookie(&hauth, "Proxy-Authorization:",
+				   extra_header, &HTTP_proxy_parsed, &hr);
 		proxy_auth_cookie = ss;
 		if (ss == NULL) {
 		    /* abort */
diff --git a/proto.h b/proto.h
index 2d36b47..f74797d 100644
--- a/proto.h
+++ b/proto.h
@@ -1,4 +1,4 @@
-/* $Id: proto.h,v 1.29 2002/01/22 16:59:11 ukai Exp $ */
+/* $Id: proto.h,v 1.30 2002/01/29 19:08:50 ukai Exp $ */
 /* 
  *   This file was automatically generated by version 1.7 of cextract.
  *   Manual editing not recommended.
@@ -416,10 +416,12 @@ extern void parseURL2(char *url, ParsedURL *pu, ParsedURL *current);
 extern Str parsedURL2Str(ParsedURL *pu);
 extern int getURLScheme(char **url);
 extern void init_stream(URLFile *uf, int scheme, InputStream stream);
+Str HTTPrequestMethod(HRequest *hr);
+Str HTTPrequestURI(ParsedURL *pu, HRequest *hr);
 extern URLFile openURL(char *url, ParsedURL *pu, ParsedURL *current,
 		       URLOption *option, FormList *request,
 		       TextList *extra_header, URLFile *ouf,
-		       unsigned char *status);
+		       HRequest *hr, unsigned char *status);
 extern int mailcapMatch(struct mailcap *mcap, char *type);
 extern struct mailcap *searchMailcap(struct mailcap *table, char *type);
 extern void initMailcap();
diff --git a/url.c b/url.c
index 50931b8..df6c16d 100644
--- a/url.c
+++ b/url.c
@@ -1,4 +1,4 @@
-/* $Id: url.c,v 1.38 2002/01/21 18:34:00 ukai Exp $ */
+/* $Id: url.c,v 1.39 2002/01/29 19:08:50 ukai Exp $ */
 #include "fm.h"
 #include <sys/types.h>
 #include <sys/socket.h>
@@ -1236,28 +1236,29 @@ otherinfo(ParsedURL *target, ParsedURL *current, char *referer)
     return s->ptr;
 }
 
-static Str
-HTTPrequest(ParsedURL *pu, ParsedURL *current, HRequest *hr, TextList *extra)
+Str
+HTTPrequestMethod(HRequest *hr)
 {
-    Str tmp;
-    TextListItem *i;
-#ifdef USE_COOKIE
-    Str cookie;
-#endif				/* USE_COOKIE */
     switch (hr->command) {
     case HR_COMMAND_CONNECT:
-	tmp = Strnew_charp("CONNECT ");
-	break;
+	return Strnew_charp("CONNECT");
     case HR_COMMAND_POST:
-	tmp = Strnew_charp("POST ");
+	return Strnew_charp("POST");
 	break;
     case HR_COMMAND_HEAD:
-	tmp = Strnew_charp("HEAD ");
+	return Strnew_charp("HEAD");
 	break;
     case HR_COMMAND_GET:
     default:
-	tmp = Strnew_charp("GET ");
+	return Strnew_charp("GET");
     }
+    return NULL;
+}
+
+Str
+HTTPrequestURI(ParsedURL *pu, HRequest *hr)
+{
+    Str tmp = Strnew();
     if (hr->command == HR_COMMAND_CONNECT) {
 	Strcat_charp(tmp, pu->host);
 	Strcat(tmp, Sprintf(":%d", pu->port));
@@ -1272,6 +1273,20 @@ HTTPrequest(ParsedURL *pu, ParsedURL *current, HRequest *hr, TextList *extra)
     else {
 	Strcat(tmp, _parsedURL2Str(pu, TRUE));
     }
+    return tmp;
+}
+
+static Str
+HTTPrequest(ParsedURL *pu, ParsedURL *current, HRequest *hr, TextList *extra)
+{
+    Str tmp;
+    TextListItem *i;
+#ifdef USE_COOKIE
+    Str cookie;
+#endif				/* USE_COOKIE */
+    tmp = HTTPrequestMethod(hr);
+    Strcat_charp(tmp, " ");
+    Strcat_charp(tmp, HTTPrequestURI(pu, hr)->ptr);
     Strcat_charp(tmp, " HTTP/1.0\r\n");
     if (hr->referer == NO_REFERER)
 	Strcat_charp(tmp, otherinfo(pu, NULL, NULL));
@@ -1346,13 +1361,13 @@ openFTPStream(ParsedURL *pu)
 URLFile
 openURL(char *url, ParsedURL *pu, ParsedURL *current,
 	URLOption *option, FormList *request, TextList *extra_header,
-	URLFile *ouf, unsigned char *status)
+	URLFile *ouf, HRequest *hr, unsigned char *status)
 {
     Str tmp;
     int i, sock, scheme;
     char *p, *q, *u;
     URLFile uf;
-    HRequest hr;
+    HRequest hr0;
 #ifdef USE_SSL
     SSL *sslh = NULL;
 #endif				/* USE_SSL */
@@ -1363,6 +1378,9 @@ openURL(char *url, ParsedURL *pu, ParsedURL *current,
 #endif				/* USE_NNTP */
     int extlen = strlen(CGI_EXTENSION);
 
+    if (hr == NULL)
+	hr = &hr0;
+
     if (ouf) {
 	uf = *ouf;
     }
@@ -1400,10 +1418,10 @@ openURL(char *url, ParsedURL *pu, ParsedURL *current,
     pu->is_nocache = (option->flag & RG_NOCACHE);
     uf.ext = filename_extension(pu->file, 1);
 
-    hr.command = HR_COMMAND_GET;
-    hr.flag = 0;
-    hr.referer = option->referer;
-    hr.request = request;
+    hr->command = HR_COMMAND_GET;
+    hr->flag = 0;
+    hr->referer = option->referer;
+    hr->request = request;
 
     switch (pu->scheme) {
     case SCM_LOCAL:
@@ -1501,7 +1519,7 @@ openURL(char *url, ParsedURL *pu, ParsedURL *current,
 	    if (sock < 0)
 		return uf;
 	    uf.scheme = SCM_HTTP;
-	    tmp = HTTPrequest(pu, current, &hr, extra_header);
+	    tmp = HTTPrequest(pu, current, hr, extra_header);
 	    write(sock, tmp->ptr, tmp->length);
 	}
 	else {
@@ -1517,9 +1535,9 @@ openURL(char *url, ParsedURL *pu, ParsedURL *current,
 	if (pu->file == NULL)
 	    pu->file = allocStr("/", -1);
 	if (request && request->method == FORM_METHOD_POST && request->body)
-	    hr.command = HR_COMMAND_POST;
+	    hr->command = HR_COMMAND_POST;
 	if (request && request->method == FORM_METHOD_HEAD)
-	    hr.command = HR_COMMAND_HEAD;
+	    hr->command = HR_COMMAND_HEAD;
 	if (non_null(HTTP_proxy) &&
 	    !Do_not_use_proxy &&
 	    pu->host != NULL && !check_no_proxy(pu->host)) {
@@ -1554,20 +1572,20 @@ openURL(char *url, ParsedURL *pu, ParsedURL *current,
 #ifdef USE_SSL
 	    if (pu->scheme == SCM_HTTPS) {
 		if (*status == HTST_NORMAL) {
-		    hr.command = HR_COMMAND_CONNECT;
-		    tmp = HTTPrequest(pu, current, &hr, NULL);
+		    hr->command = HR_COMMAND_CONNECT;
+		    tmp = HTTPrequest(pu, current, hr, NULL);
 		    *status = HTST_CONNECT;
 		}
 		else {
-		    hr.flag |= HR_FLAG_LOCAL;
-		    tmp = HTTPrequest(pu, current, &hr, extra_header);
+		    hr->flag |= HR_FLAG_LOCAL;
+		    tmp = HTTPrequest(pu, current, hr, extra_header);
 		    *status = HTST_NORMAL;
 		}
 	    }
 	    else
 #endif				/* USE_SSL */
 	    {
-		tmp = HTTPrequest(pu, current, &hr, extra_header);
+		tmp = HTTPrequest(pu, current, hr, extra_header);
 		*status = HTST_NORMAL;
 		pu->label = save_label;
 	    }
@@ -1587,8 +1605,8 @@ openURL(char *url, ParsedURL *pu, ParsedURL *current,
 		}
 	    }
 #endif				/* USE_SSL */
-	    hr.flag |= HR_FLAG_LOCAL;
-	    tmp = HTTPrequest(pu, current, &hr, extra_header);
+	    hr->flag |= HR_FLAG_LOCAL;
+	    tmp = HTTPrequest(pu, current, hr, extra_header);
 	    *status = HTST_NORMAL;
 	}
 #ifdef USE_SSL
@@ -1598,7 +1616,7 @@ openURL(char *url, ParsedURL *pu, ParsedURL *current,
 		SSL_write(sslh, tmp->ptr, tmp->length);
 	    else
 		write(sock, tmp->ptr, tmp->length);
-	    if (hr.command == HR_COMMAND_POST &&
+	    if (hr->command == HR_COMMAND_POST &&
 		request->enctype == FORM_ENCTYPE_MULTIPART) {
 		if (sslh)
 		    SSL_write_from_file(sslh, request->body);
@@ -1618,7 +1636,7 @@ openURL(char *url, ParsedURL *pu, ParsedURL *current,
 		fclose(ff);
 	    }
 #endif				/* HTTP_DEBUG */
-	    if (hr.command == HR_COMMAND_POST &&
+	    if (hr->command == HR_COMMAND_POST &&
 		request->enctype == FORM_ENCTYPE_MULTIPART)
 		write_from_file(sock, request->body);
 	}
@@ -1634,7 +1652,7 @@ openURL(char *url, ParsedURL *pu, ParsedURL *current,
 	    if (sock < 0)
 		return uf;
 	    uf.scheme = SCM_HTTP;
-	    tmp = HTTPrequest(pu, current, &hr, extra_header);
+	    tmp = HTTPrequest(pu, current, hr, extra_header);
 	}
 	else {
 	    sock = openSocket(pu->host,