Index: lib/http_client.c
===================================================================
--- lib/http_client.c	(revision 398b2533afe5860e1b83edb0fcf5d7cf80207162)
+++ lib/http_client.c	(revision 8bd866f3d336cb695006bc48592cf7475139aa36)
@@ -193,10 +193,11 @@
 }
 
+static gboolean http_handle_headers( struct http_request *req );
+
 static gboolean http_incoming_data( gpointer data, int source, b_input_condition cond )
 {
 	struct http_request *req = data;
-	int evil_server = 0;
 	char buffer[2048];
-	char *end1, *end2, *s;
+	char *s;
 	size_t content_length;
 	int st;
@@ -218,10 +219,10 @@
 				   packets that abort connections! \o/ */
 				
-				goto got_reply;
+				goto eof;
 			}
 		}
 		else if( st == 0 )
 		{
-			goto got_reply;
+			goto eof;
 		}
 	}
@@ -239,14 +240,56 @@
 		else if( st == 0 )
 		{
-			goto got_reply;
-		}
-	}
-	
-	if( st > 0 )
+			goto eof;
+		}
+	}
+	
+	if( st > 0 && !req->sbuf )
 	{
 		req->reply_headers = g_realloc( req->reply_headers, req->bytes_read + st + 1 );
 		memcpy( req->reply_headers + req->bytes_read, buffer, st );
 		req->bytes_read += st;
-	}
+		
+		st = 0;
+	}
+	
+	if( st >= 0 && ( req->flags & HTTPC_STREAMING ) )
+	{
+		if( !req->reply_body &&
+		    ( strstr( req->reply_headers, "\r\n\r\n" ) ||
+		      strstr( req->reply_headers, "\n\n" ) ) )
+		{
+			size_t hlen;
+			
+			/* We've now received all headers, so process them once
+			   before we start feeding back data. */
+			if( !http_handle_headers( req ) )
+				return FALSE;
+			
+			hlen = req->reply_body - req->reply_headers;
+			
+			req->sblen = req->bytes_read - hlen;
+			req->sbuf = g_memdup( req->reply_body, req->sblen + 1 );
+			req->reply_headers = g_realloc( req->reply_headers, hlen + 1 );
+			
+			req->reply_body = req->sbuf;
+		}
+		
+		if( st > 0 )
+		{
+			int pos = req->reply_body - req->sbuf;
+			req->sbuf = g_realloc( req->sbuf, req->sblen + st + 1 );
+			memcpy( req->sbuf + req->sblen, buffer, st );
+			req->bytes_read += st;
+			req->sblen += st;
+			req->reply_body = req->sbuf + pos;
+			req->body_size = req->sblen - pos;
+		}
+		
+		if( req->reply_body )
+			req->func( req );
+	}
+	
+	if( ssl_pending( req->ssl ) )
+		return http_incoming_data( data, source, cond );
 	
 	/* There will be more! */
@@ -255,10 +298,7 @@
 	                         http_incoming_data, req );
 	
-	if( ssl_pending( req->ssl ) )
-		return http_incoming_data( data, source, cond );
-	else
-		return FALSE;
-
-got_reply:
+	return FALSE;
+
+eof:
 	/* Maybe if the webserver is overloaded, or when there's bad SSL
 	   support... */
@@ -269,6 +309,48 @@
 	}
 	
+	if( !( req->flags & HTTPC_STREAMING ) )
+	{
+		/* Returns FALSE if we were redirected, in which case we should abort
+		   and not run any callback yet. */
+		if( !http_handle_headers( req ) )
+			return FALSE;
+	}
+
+cleanup:
+	if( req->ssl )
+		ssl_disconnect( req->ssl );
+	else
+		closesocket( req->fd );
+	
+	if( ( s = get_rfc822_header( req->reply_headers, "Content-Length", 0 ) ) &&
+	    sscanf( s, "%zd", &content_length ) == 1 )
+	{
+		if( content_length < req->body_size )
+		{
+			req->status_code = -1;
+			g_free( req->status_string );
+			req->status_string = g_strdup( "Response truncated" );
+		}
+	}
+	g_free( s );
+	
+	if( getenv( "BITLBEE_DEBUG" ) && req )
+		printf( "Finishing HTTP request with status: %s\n",
+		        req->status_string ? req->status_string : "NULL" );
+	
+	req->func( req );
+	http_free( req );
+	return FALSE;
+}
+
+/* Splits headers and body. Checks result code, in case of 300s it'll handle
+   redirects. If this returns FALSE, don't call any callbacks! */
+static gboolean http_handle_headers( struct http_request *req )
+{
+	char *end1, *end2;
+	int evil_server = 0;
+	
 	/* Zero termination is very convenient. */
-	req->reply_headers[req->bytes_read] = 0;
+	req->reply_headers[req->bytes_read] = '\0';
 	
 	/* Find the separation between headers and body, and keep stupid
@@ -289,5 +371,5 @@
 	{
 		req->status_string = g_strdup( "Malformed HTTP reply" );
-		goto cleanup;
+		return TRUE;
 	}
 	
@@ -306,5 +388,5 @@
 	if( ( end1 = strchr( req->reply_headers, ' ' ) ) != NULL )
 	{
-		if( sscanf( end1 + 1, "%d", &req->status_code ) != 1 )
+		if( sscanf( end1 + 1, "%hd", &req->status_code ) != 1 )
 		{
 			req->status_string = g_strdup( "Can't parse status code" );
@@ -349,5 +431,5 @@
 		{
 			req->status_string = g_strdup( "Can't locate Location: header" );
-			goto cleanup;
+			return TRUE;
 		}
 		
@@ -369,5 +451,5 @@
 			req->status_string = g_strdup( "Can't handle recursive redirects" );
 			
-			goto cleanup;
+			return TRUE;
 		}
 		else
@@ -380,5 +462,5 @@
 			s = strstr( loc, "\r\n" );
 			if( s == NULL )
-				goto cleanup;
+				return TRUE;
 			
 			url = g_new0( url_t, 1 );
@@ -389,5 +471,5 @@
 				req->status_string = g_strdup( "Malformed redirect URL" );
 				g_free( url );
-				goto cleanup;
+				return TRUE;
 			}
 			
@@ -401,5 +483,5 @@
 				req->status_string = g_strdup( "Error while rebuilding request string" );
 				g_free( url );
-				goto cleanup;
+				return TRUE;
 			}
 			
@@ -467,5 +549,5 @@
 			req->status_string = g_strdup( "Connection problem during redirect" );
 			g_free( new_request );
-			goto cleanup;
+			return TRUE;
 		}
 		
@@ -480,33 +562,14 @@
 	}
 	
-	/* Assume that a closed connection means we're finished, this indeed
-	   breaks with keep-alive connections and faulty connections. */
-	/* req->finished = 1; */
-
-cleanup:
-	if( req->ssl )
-		ssl_disconnect( req->ssl );
-	else
-		closesocket( req->fd );
-	
-	if( ( s = get_rfc822_header( req->reply_headers, "Content-Length", 0 ) ) &&
-	    sscanf( s, "%zd", &content_length ) == 1 )
-	{
-		if( content_length < req->body_size )
-		{
-			req->status_code = -1;
-			g_free( req->status_string );
-			req->status_string = g_strdup( "Response truncated" );
-		}
-	}
-	g_free( s );
-	
-	if( getenv( "BITLBEE_DEBUG" ) && req )
-		printf( "Finishing HTTP request with status: %s\n",
-		        req->status_string ? req->status_string : "NULL" );
-	
-	req->func( req );
-	http_free( req );
-	return FALSE;
+	return TRUE;
+}
+
+void http_flush_bytes( struct http_request *req, size_t len )
+{
+	if( len > 0 && len <= req->body_size )
+	{
+		req->reply_body += len;
+		req->body_size -= len;
+	}
 }
 
@@ -516,5 +579,5 @@
 	g_free( req->reply_headers );
 	g_free( req->status_string );
+	g_free( req->sbuf );
 	g_free( req );
 }
-
Index: lib/http_client.h
===================================================================
--- lib/http_client.h	(revision 398b2533afe5860e1b83edb0fcf5d7cf80207162)
+++ lib/http_client.h	(revision 8bd866f3d336cb695006bc48592cf7475139aa36)
@@ -26,15 +26,10 @@
 /* http_client allows you to talk (asynchronously, again) to HTTP servers.
    In the "background" it will send the whole query and wait for a complete
-   response to come back. Right now it's only used by the MSN Passport
-   authentication code, but it might be useful for other things too (for
-   example the AIM usericon patch uses this so icons can be stored on
-   webservers instead of the local filesystem).
+   response to come back. Initially written for MS Passport authentication,
+   but used for many other things now like OAuth and Twitter.
    
-   Didn't test this too much, but it seems to work well. Just don't look
-   at the code that handles HTTP 30x redirects. ;-) The function is
-   probably not very useful for downloading lots of data since it keeps 
-   everything in a memory buffer until the download is completed (and
-   can't pass any data or whatever before then). It's very useful for
-   doing quick requests without blocking the whole program, though. */
+   It's very useful for doing quick requests without blocking the whole
+   program. Unfortunately it doesn't support fancy stuff like HTTP keep-
+   alives. */
 
 #include <glib.h>
@@ -42,4 +37,9 @@
 
 struct http_request;
+
+typedef enum http_client_flags
+{
+	HTTPC_STREAMING = 1,
+} http_client_flags_t;
 
 /* Your callback function should look like this: */
@@ -53,5 +53,5 @@
 	char *request;          /* The request to send to the server. */
 	int request_length;     /* Its size. */
-	int status_code;        /* The numeric HTTP status code. (Or -1
+	short status_code;      /* The numeric HTTP status code. (Or -1
 	                           if something really went wrong) */
 	char *status_string;    /* The error text. */
@@ -59,8 +59,8 @@
 	char *reply_body;
 	int body_size;          /* The number of bytes in reply_body. */
-	/* int finished;           Set to non-0 if the request was completed
-	                           successfully. */
-	int redir_ttl;          /* You can set it to 0 if you don't want
+	short redir_ttl;        /* You can set it to 0 if you don't want
 	                           http_client to follow them. */
+	
+	http_client_flags_t flags;
 	
 	http_input_function func;
@@ -68,5 +68,4 @@
 	
 	/* Please don't touch the things down here, you shouldn't need them. */
-	
 	void *ssl;
 	int fd;
@@ -75,4 +74,8 @@
 	int bytes_written;
 	int bytes_read;
+	
+	/* Used in streaming mode. Caller should read from reply_body. */
+	char *sbuf;
+	size_t sblen;
 };
 
@@ -83,2 +86,5 @@
 struct http_request *http_dorequest( char *host, int port, int ssl, char *request, http_input_function func, gpointer data );
 struct http_request *http_dorequest_url( char *url_string, http_input_function func, gpointer data );
+
+/* For streaming connections only; flushes len bytes at the start of the buffer. */
+void http_flush_bytes( struct http_request *req, size_t len );
