[vlc-devel] [PATCH] network: implement an API for HTTP requests.
Rémi Denis-Courmont
remi at remlab.net
Mon Feb 3 19:30:24 CET 2014
On Monday 03 February 2014 19:07:48 Felix Abecassis wrote:
> We would like to provide a simple API for sending HTTP requests and
> for parsing the response from the server. Many modules are currently
> handcrafting GET/POST requests by manipulating strings.
>
> Comments are welcome concerning the API and the features that should
> be implemented.
>
> Difference from last draft:
> - Now correctly split headers.
> - Now automatically use Expect: 100-continue for large request bodies
> if HTTP version is 1.1.
> - Moved declaration to vlc_http.h and definition to a new file.
>
> To discuss:
> - HTTP 2.0: no official standard yet. What should we do?
Support for reusing the same connection across requests. This would be useful
for HTTP 1.1 too anyway.
> - For courmisch: you've mentioned special FD polling in LUA. Could you give
> me more details?
All I am saying is that you cannot just block the calling thread with no way
out. There are two separate underlying issues. First you need to time out if
the server does not answer in a reasonable time line. Second, the calling
thread needs a way to not get stuck for an arbitrarily long time if it needs
to abort (such as because VLC is quitting).
How you solve this issue is an implementation choice. The current Lua net code
is just one solution, that fits perhaps relatively well with the Lua
interpreter. Here you have different constraints, since the API is not tied to
Lua.
> --
> include/vlc_http.h | 68 ++++++++++
> modules/misc/Modules.am | 4 +
> src/Makefile.am | 1 +
> src/libvlccore.sym | 2 +
> src/network/http.c | 349
> ++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 424
> insertions(+)
> create mode 100644 src/network/http.c
>
> diff --git a/include/vlc_http.h b/include/vlc_http.h
> index ddde13e..a64e61f 100644
> --- a/include/vlc_http.h
> +++ b/include/vlc_http.h
> @@ -64,4 +64,72 @@ VLC_API char *http_auth_FormatAuthorizationHeader
> const char *, const char *,
> const char *, const char * ) VLC_USED;
>
> +typedef enum
> +{
> + VLC_HTTP_OPTIONS,
> + VLC_HTTP_GET,
> + VLC_HTTP_HEAD,
> + VLC_HTTP_POST,
> + VLC_HTTP_PUT,
> + VLC_HTTP_DELETE,
> + VLC_HTTP_TRACE,
> + VLC_HTTP_CONNECT,
> + VLC_HTTP_PATCH
> +} vlc_http_method_t;
> +
> +typedef struct
> +{
> + vlc_http_method_t method;
> + const char* uri;
> + int http_version;
> +} vlc_http_request_line_t;
> +
> +typedef struct
> +{
> + const char* key;
> + const char* value;
> +} vlc_http_header_pair_t;
> +
> +/* Headers are stored as a dictionary.
> + For instance 'User-agent: foo' is stored as { "User-agent", "foo" }.
> +*/
> +typedef struct
> +{
> + vlc_http_header_pair_t *dictionary;
> + size_t size;
> +} vlc_http_header_t;
> +
> +typedef struct
> +{
> + void* data;
> + size_t size;
> +} vlc_http_body_t;
> +
> +typedef struct
> +{
> + vlc_http_request_line_t request_line;
> + vlc_http_header_t header;
> + vlc_http_body_t body;
> +} vlc_http_request_t;
> +
> +typedef struct
> +{
> + int http_version;
> + int status_code;
> + /* Reason-Phrase not saved. */
> +} vlc_http_status_line_t;
> +
> +typedef struct
> +{
> + vlc_http_status_line_t status_line;
> + vlc_http_header_t header;
> + vlc_http_body_t body;
> +} vlc_http_response_t;
> +
> +VLC_API void http_response_Delete(vlc_http_response_t *response);
> +
> +VLC_API vlc_http_response_t *http_request_Send(vlc_object_t *p_this,
> vlc_url_t *url, +
> vlc_http_request_t *request); +#define http_request_Send(a,b,c)
> http_request_Send(VLC_OBJECT(a),b,c) +
> #endif /* VLC_HTTP_H */
> diff --git a/modules/misc/Modules.am b/modules/misc/Modules.am
> index f0b69d4..dd71f28 100644
> --- a/modules/misc/Modules.am
> +++ b/modules/misc/Modules.am
> @@ -6,6 +6,10 @@ libaudioscrobbler_plugin_la_SOURCES = audioscrobbler.c
> libaudioscrobbler_plugin_la_LIBADD = $(SOCKET_LIBS) $(LIBPTHREAD)
> misc_LTLIBRARIES += libaudioscrobbler_plugin.la
>
> +libhttprequest_plugin_la_SOURCES = httprequest.c
> +libhttprequest_plugin_la_LIBADD = $(SOCKET_LIBS) $(LIBPTHREAD)
> +misc_LTLIBRARIES += libhttprequest_plugin.la
> +
> libexport_plugin_la_SOURCES = \
> playlist/html.c \
> playlist/m3u.c \
> diff --git a/src/Makefile.am b/src/Makefile.am
> index 474da25..9d46adf 100644
> --- a/src/Makefile.am
> +++ b/src/Makefile.am
> @@ -424,6 +424,7 @@ SOURCES_libvlc_common = \
> network/udp.c \
> network/rootbind.c \
> network/tls.c \
> + network/http.c \
> text/charset.c \
> text/strings.c \
> text/unicode.c \
> diff --git a/src/libvlccore.sym b/src/libvlccore.sym
> index 3fc39d7..6fd95d8 100644
> --- a/src/libvlccore.sym
> +++ b/src/libvlccore.sym
> @@ -144,6 +144,8 @@ http_auth_Reset
> http_auth_ParseWwwAuthenticateHeader
> http_auth_ParseAuthenticationInfoHeader
> http_auth_FormatAuthorizationHeader
> +http_request_Send
> +http_response_Delete
> httpd_ClientIP
> httpd_FileDelete
> httpd_FileNew
> diff --git a/src/network/http.c b/src/network/http.c
> new file mode 100644
> index 0000000..8326533
> --- /dev/null
> +++ b/src/network/http.c
> @@ -0,0 +1,349 @@
> +/**************************************************************************
> *** + * http.c: functions to send HTTP requests and receive responses. +
> ***************************************************************************
> ** + * Copyright (C) 2014 VLC authors and VideoLAN
> + *
> + * Authors: Felix Abecassis <felix.abecassis at gmail.com>
> + *
> + * This program is free software; you can redistribute it and/or modify it
> + * under the terms of the GNU Lesser General Public License as published by
> + * the Free Software Foundation; either version 2.1 of the License, or + *
> (at your option) any later version.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> + * GNU Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public License
> + * along with this program; if not, write to the Free Software Foundation,
> + * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA. +
> ***************************************************************************
> **/ +
> +#ifdef HAVE_CONFIG_H
> +# include "config.h"
> +#endif
> +
> +#include <vlc_common.h>
> +#include <vlc_url.h>
> +#include <vlc_network.h>
> +#include <vlc_http.h>
> +
> +#include <errno.h>
> +
> +/* Read a line (ended by \r\n) from an HTTP response. */
> +static char* http_readline( vlc_object_t *p_this, int fd )
> +{
> + size_t size = 64;
> + char *str = malloc(sizeof(*str) * size);
> +
> + char c;
> + size_t i = 0;
> + while (net_Read(p_this, fd, NULL, &c, 1, false) > 0)
This can easily get stuck forever. Same in just about every other send or
receive calls down...
> + {
> + if (i == size - 1)
> + {
> + /* Ran out of space for storing the line, grow the buffer. */
> + size *= 2;
> + str = realloc(str, sizeof(*str) * size);
> + if (!str)
> + return NULL;
> + }
> + str[i++] = c;
> + if (c == '\n')
> + break;
> + }
> +
> + if (i == 0)
> + {
> + free(str);
> + return NULL;
> + }
> +
> + str[i] = '\0';
> + return str;
> +}
> +
> +/* Concatenate the given key-value pair to string headers. */
> +static char *http_add_header(char *headers, const char *key, const char
> *value) +{
> + char *new_headers;
> + int ret = asprintf(&new_headers, "%s%s: %s\r\n", headers, key, value);
> + free(headers);
> + return ret >= 0 ? new_headers : NULL;
> +}
> +
> +/* Split a header line with format "Key: Value" */
> +static int http_split_header_line(const char *line, char** key, char
> **value) +{
> + *key = NULL;
> + *value = NULL;
> +
> + const char *key_begin = line;
> + const char *key_end = strchr(key_begin, ':');
> + *key = strndup(key_begin, key_end - key_begin);
> + if (!*key)
> + goto error;
> +
> + const char *value_begin = key_end + 1;
> + while (*value_begin == ' ')
> + ++value_begin;
> + const char *value_end = strchr(value_begin, '\r');
> + if (!value_end)
> + goto error;
> + /* If value_end == value_begin, the field is empty, this is valid. */
> + if (value_end != value_begin)
> + {
> + *value = strndup(value_begin, value_end - value_begin);
> + if (!*value)
> + goto error;
> + }
> +
> + return 0;
> +
> +error:
> + free(*key);
> + free(*value);
> + return -1;
> +}
> +
> +static vlc_http_response_t *http_get_response(vlc_object_t *p_this, int fd)
> +{
> + vlc_http_response_t *response = calloc(1, sizeof(*response));
> + if (!response)
> + return NULL;
> +
> + vlc_http_status_line_t *status_line = &response->status_line;
> + vlc_http_header_t *header = &response->header;
> + vlc_http_body_t *body = &response->body;
> +
> + /* Parse status line. */
> + char *line = http_readline(p_this, fd);
> + int ret = sscanf(line, "HTTP/1.%d %3d", &status_line->http_version,
> &status_line->status_code);
No, this is more restrictive than the specified HTTP syntax.
> + free(line);
> + if (ret != 2)
> + {
> + msg_Err(p_this, "Received invalid HTTP status line.");
What if the response is redirected? What if the server is version 0.9?
> + goto error;
> + }
> +
> + /* Parse headers, line by line. */
> + while ((line = http_readline(p_this, fd)))
> + {
> + /* Line with only \r\n: end of headers. */
> + if (!strncmp(line, "\r\n", 2))
> + {
> + free(line);
> + break;
> + }
> +
> + /* Extract the key-value pair from the raw line. */
> + char *key = NULL;
> + char *value = NULL;
> + ret = http_split_header_line(line, &key, &value);
> + free(line);
> + if (ret < 0)
> + goto error;
> +
> + /* Grow the dictionary and add the new key-value pair. */
> + header->dictionary = realloc(header->dictionary,
> sizeof(*header->dictionary) * (header->size + 1)); + if
> (!header->dictionary)
> + {
> + free(key);
> + free(value);
> + goto error;
> + }
> + header->dictionary[header->size].key = key;
> + header->dictionary[header->size].value = value;
> + ++header->size;
> + }
> +
> + /* Try to find Content-Length entry in the header */
> + for (size_t i = 0; i < header->size; ++i)
> + {
> + const char *key = header->dictionary[i].key;
> + const char *value = header->dictionary[i].value;
> + if (!strncasecmp(key, "Content-Length", 14))
> + {
> + /* Get the field value. */
> + errno = 0;
> + body->size = strtoul(value, NULL, 10);
> + if (errno)
> + {
> + msg_Err(p_this, "Invalid Content-Length in HTTP
> response."); + goto error;
> + }
> + break;
> + }
> + }
> +
> + /* Read the body of the response. */
> + if (body->size > 0)
What if the body size is not specified (i.e. no Content-Length)?
Also what if the body size is insanely large?
> + {
> + /* We add a terminal \0 for debugging convenience. */
> + body->data = malloc(body->size + 1);
> + if (!body->data)
> + goto error;
> +
> + size_t size = net_Read(p_this, fd, NULL, body->data, body->size,
> true);
> + if (size < body->size)
> + goto error;
> +
> + char* end = (char*)body->data + body->size;
What about transfer encoding?
With all those mistakes, I have to believe you did not read RFC2616...
> + *end = '\0';
> + }
> + return response;
> +
> +error:
> + msg_Err(p_this, "Error in %s", __FUNCTION__);
> + http_response_Delete(response);
> + return NULL;
> +}
> +
> +typedef struct http_method_s
> +{
> + vlc_http_method_t id;
> + const char *string;
> +} http_method_t;
> +
> +static const struct http_method_s http_methods[] =
> +{
> + { VLC_HTTP_OPTIONS, "OPTIONS" },
> + { VLC_HTTP_GET, "GET" },
> + { VLC_HTTP_HEAD, "HEAD" },
> + { VLC_HTTP_POST, "POST" },
> + { VLC_HTTP_PUT, "PUT" },
> + { VLC_HTTP_DELETE, "DELETE" },
> + { VLC_HTTP_TRACE, "TRACE" },
> + { VLC_HTTP_CONNECT, "CONNECT" },
> + { VLC_HTTP_PATCH, "PATCH" },
> +};
> +#define HTTP_METHODS_COUNT (sizeof(http_methods)/sizeof(http_methods[0]))
> +
> +#define HTTP_EXPECT_CONTINUE_THRESHOLD 1024
> +
> +#undef http_request_Send
> +vlc_http_response_t *http_request_Send(vlc_object_t *p_this, vlc_url_t
> *url, + vlc_http_request_t *request)
> +{
> + vlc_http_request_line_t *request_line = &request->request_line;
> + vlc_http_header_t *request_header = &request->header;
> + vlc_http_body_t *request_body = &request->body;
> +
> + char *header_string = NULL;
> + vlc_http_response_t *response = NULL;
> +
> + int fd = net_ConnectTCP(p_this, url->psz_host, url->i_port);
> + if (fd < 0)
> + {
> + msg_Err(p_this, "Could not connect to %s:%d", url->psz_host,
> url->i_port); + return NULL;
> + }
These days, I don't think support for TLS can be considered optional.
> +
> + /* Lookup the string corresponding to the HTTP method. */
> + const char* method_string = NULL;
> + for (unsigned i = 0; i < HTTP_METHODS_COUNT; ++i)
> + {
> + if (http_methods[i].id == request_line->method)
> + {
> + method_string = http_methods[i].string;
> + break;
> + }
> + }
> + if (!method_string)
> + goto end;
> +
> + /* Create the header string. */
> + int ret;
> + header_string = strdup("");
> + for (size_t i = 0; i < request_header->size; ++i)
> + {
> + const char *key = request_header->dictionary[i].key;
> + const char *value = request_header->dictionary[i].value;
> + /* Discard unauthorized headers */
> + if (!strncasecmp(key, "Content-Length", 14)
> + || !strncasecmp(key, "Connection", 10))
Those are presumably not the only unsafe headers.
> + {
> + msg_Err(p_this, "Unauthorized HTTP header: %s %s (overwritten
> by VLC)", key, value); + continue;
> + }
> + /* Concatenate new key-value entry to the string. */
> + header_string = http_add_header(header_string, key, value);
> + if (!header_string)
> + goto end;
> + }
> + /* If the body is larger than the threshold, we first ask the
> + server if the headers are acceptable using Expect: 100-continue.
> + This is only for HTTP 1.1. */
> + bool use_expect_continue = request_line->http_version >= 1 &&
> request_body->size >= HTTP_EXPECT_CONTINUE_THRESHOLD; + if
> (use_expect_continue)
> + if (!(header_string = http_add_header(header_string, "Expect",
> "100-continue"))) + goto end;
> +
> + /* Force "Connection: close" mode. */
> + if (!(header_string = http_add_header(header_string, "Connection",
> "close"))) + goto end;
> +
> + /* Send request line and header. */
> + ret = net_Printf(p_this, fd, NULL,
> + "%s %s HTTP/1.%d\r\n"
> + "%s"
> + "Content-Length: %zu\r\n"
> + "\r\n",
> + method_string, request_line->uri,
> request_line->http_version, + header_string,
> request_body->size);
> + if (ret == -1)
> + goto end;
> +
> + if (use_expect_continue)
> + {
> + vlc_http_response_t *expect_response = http_get_response(p_this,
> fd); + if (!expect_response) /* Error while reading the response. */
> + goto end;
> + int status_code = expect_response->status_line.status_code;
> + http_response_Delete(expect_response);
> + if (status_code != 100) /* Server did not accept the headers. */
> + {
> + msg_Err(p_this, "\"Expect: 100-continue\" request denied
> (status: %d).", status_code); + goto end;
> + }
> + }
> +
> + /* Send body. */
> + if (request_body->size > 0)
> + {
> + ret = net_Write(p_this, fd, NULL, request_body->data,
> request_body->size); + if (ret == -1)
> + {
> + msg_Err(p_this, "Could not send HTTP request body.");
> + goto end;
> + }
> + }
> +
> + response = http_get_response(p_this, fd);
> +
> +end:
> + free(header_string);
> + net_Close(fd);
> + return response;
> +}
> +
> +void http_response_Delete(vlc_http_response_t *response)
> +{
> + if (!response)
> + return;
> +
> + vlc_http_header_t *header = &response->header;
> + vlc_http_body_t *body = &response->body;
> +
> + for (size_t i = 0; i < header->size; ++i)
> + {
> + char *key = (char*)header->dictionary[i].key;
> + char *value = (char*)header->dictionary[i].value;
> + free(key);
> + free(value);
> + }
> + free(header->dictionary);
> + free(body->data);
> + free(response);
> +}
--
Rémi Denis-Courmont
http://www.remlab.net/
More information about the vlc-devel
mailing list