[vlc-devel] [PATCH 10/10][RFC][WIP] cloudstorage: replaced curl dependency by using functions from http module
Diogo Silva
dbtdsilva at gmail.com
Thu Aug 17 03:08:18 CEST 2017
This patch includes a stream that is responsible to receive an istream
from libcloudstorage and stream it block by block using block_t and the
already existing vlc_http_stream.
---
modules/access/Makefile.am | 6 +-
modules/access/cloudstorage/access.cpp | 3 +-
modules/access/cloudstorage/provider_http.cpp | 289 +++++++++++++++++++++
modules/access/cloudstorage/provider_http.h | 89 +++++++
.../access/cloudstorage/provider_http_stream.cpp | 96 +++++++
modules/access/cloudstorage/provider_http_stream.h | 30 +++
po/POTFILES.in | 4 +
7 files changed, 515 insertions(+), 2 deletions(-)
create mode 100644 modules/access/cloudstorage/provider_http.cpp
create mode 100644 modules/access/cloudstorage/provider_http.h
create mode 100644 modules/access/cloudstorage/provider_http_stream.cpp
create mode 100644 modules/access/cloudstorage/provider_http_stream.h
diff --git a/modules/access/Makefile.am b/modules/access/Makefile.am
index 32b80afa2b..036c909575 100644
--- a/modules/access/Makefile.am
+++ b/modules/access/Makefile.am
@@ -319,10 +319,14 @@ libcloudstorage_plugin_la_SOURCES = \
access/cloudstorage/services_discovery.h \
access/cloudstorage/services_discovery.cpp \
access/cloudstorage/provider_callback.h \
+ access/cloudstorage/provider_http.h \
+ access/cloudstorage/provider_http.cpp \
+ access/cloudstorage/provider_http_stream.h \
+ access/cloudstorage/provider_http_stream.cpp \
access/cloudstorage/provider_httpd.h \
access/cloudstorage/provider_httpd.cpp
libcloudstorage_plugin_la_CXXFLAGS = $(CXXFLAGS_cloudstorage) $(AM_CXXFLAGS)
-libcloudstorage_plugin_la_LIBADD = $(LIBS_cloudstorage)
+libcloudstorage_plugin_la_LIBADD = $(LIBS_cloudstorage) libvlc_http.la
libcloudstorage_plugin_la_LDFLAGS = $(AM_LDFLAGS) -rpath '$(accessdir)'
access_LTLIBRARIES += $(LTLIBcloudstorage)
EXTRA_LTLIBRARIES += libcloudstorage_plugin.la
diff --git a/modules/access/cloudstorage/access.cpp b/modules/access/cloudstorage/access.cpp
index c4780e53f6..c1d10fd336 100644
--- a/modules/access/cloudstorage/access.cpp
+++ b/modules/access/cloudstorage/access.cpp
@@ -31,6 +31,7 @@
#include "provider_callback.h"
#include "provider_httpd.h"
+#include "provider_http.h"
using cloudstorage::ICloudStorage;
using cloudstorage::ICloudProvider;
@@ -179,7 +180,7 @@ static int InitProvider( stream_t * p_access )
p_sys->token,
std::unique_ptr<Callback>( new Callback( p_access ) ),
nullptr,
- nullptr,
+ std::unique_ptr<Http>( new Http(p_access ) ),
std::unique_ptr<HttpdFactory>( new HttpdFactory( p_access ) ),
hints
});
diff --git a/modules/access/cloudstorage/provider_http.cpp b/modules/access/cloudstorage/provider_http.cpp
new file mode 100644
index 0000000000..94105f17b9
--- /dev/null
+++ b/modules/access/cloudstorage/provider_http.cpp
@@ -0,0 +1,289 @@
+/*****************************************************************************
+ * provider_http.cpp: Inherit class IHttp from libcloudstorage
+ *****************************************************************************
+ * Copyright (C) 2017 VideoLabs and VideoLAN
+ *
+ * Authors: Diogo Silva <dbtdsilva at gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
+ *****************************************************************************/
+
+#include "provider_http.h"
+
+#include <memory>
+#include <vlc_common.h>
+#include <vlc_url.h>
+#include <vlc_block.h>
+#include <json/json.h>
+#include <iterator>
+
+extern "C" {
+#include "access/http/resource.h"
+#include "access/http/connmgr.h"
+#include "access/http/message.h"
+}
+
+#include "provider_http_stream.h"
+
+#define URL_REDIRECT_LIMIT 10
+
+// Initializing static members
+const struct vlc_http_resource_cbs HttpRequest::handler_callbacks =
+ { httpRequestHandler, httpResponseHandler };
+
+// HttpRequest interface implementation
+struct HttpRequestData {
+ const HttpRequest *ptr;
+ std::istream *data;
+};
+
+HttpRequest::HttpRequest( stream_t* access, const std::string& url,
+ const std::string& method, bool follow_redirect ) :
+ p_access( access ), req_url( url ), req_method( method ),
+ req_follow_redirect( follow_redirect ),
+ manager( vlc_http_mgr_create( VLC_OBJECT(p_access), NULL ))
+{
+}
+
+HttpRequest::~HttpRequest()
+{
+ if ( manager != NULL )
+ vlc_http_mgr_destroy( manager );
+}
+
+void HttpRequest::setParameter( const std::string& parameter,
+ const std::string& value )
+{
+ req_parameters[parameter] = value;
+}
+
+void HttpRequest::setHeaderParameter( const std::string& parameter,
+ const std::string& value )
+{
+ req_header_parameters[parameter] = value;
+}
+
+const std::unordered_map<std::string, std::string>&
+ HttpRequest::parameters() const
+{
+ return req_parameters;
+}
+
+const std::unordered_map<std::string, std::string>&
+ HttpRequest::headerParameters() const
+{
+ return req_header_parameters;
+}
+
+const std::string& HttpRequest::url() const
+{
+ return req_url;
+}
+
+const std::string& HttpRequest::method() const
+{
+ return req_method;
+}
+
+bool HttpRequest::follow_redirect() const
+{
+ return req_follow_redirect;
+}
+
+void HttpRequest::send( CompleteCallback on_completed,
+ std::shared_ptr<std::istream> data,
+ std::shared_ptr<std::ostream> response,
+ std::shared_ptr<std::ostream> error_stream,
+ ICallback::Pointer cb ) const
+{
+ std::string current_url = req_url;
+ int redirect_counter = 0;
+ // default error is an internal error, it will be rewriten when it gets a
+ // request successfully
+ int response_code = 500;
+
+request:
+ struct vlc_http_resource *resource =
+ (struct vlc_http_resource *) malloc( sizeof( *resource ) );
+ if ( resource == NULL ) {
+ on_completed( response_code, response, error_stream );
+ return;
+ }
+
+ // Concatenating the URL in the parameters
+ std::string params_url;
+ std::unordered_map<std::string, std::string>::const_iterator it;
+ for ( it = req_parameters.begin(); it != req_parameters.end(); ++it )
+ {
+ if ( it != req_parameters.begin() )
+ params_url += "&";
+ params_url += it->first + "=" + it->second;
+ }
+ std::string url = current_url + (!params_url.empty() ?
+ ("?" + params_url) : "");
+
+ // Init the resource
+ // Fixup in order to fix the raw URL (not-encoded parts) from the
+ // libcloudstorage. Either way, its safer to prevent.
+ char *fixed_url = vlc_uri_fixup( url.c_str() );
+ int result = vlc_http_res_init( resource, &handler_callbacks, manager,
+ fixed_url, NULL, NULL, req_method.c_str() );
+ free(fixed_url);
+ if ( result != VLC_SUCCESS )
+ {
+ vlc_http_res_destroy( resource );
+ on_completed( response_code, response, error_stream );
+ return;
+ }
+
+ // Invoke the HTTP request (GET, POST, ..)
+ HttpRequestData *callback_data = new HttpRequestData();
+ if ( callback_data == NULL )
+ {
+ vlc_http_res_destroy( resource );
+ on_completed( response_code, response, error_stream );
+ return;
+ }
+ callback_data->ptr = this;
+ callback_data->data = data.get();
+ resource->response = vlc_http_res_open( resource, callback_data );
+ delete callback_data;
+
+ // Get the response code obtained
+ if ( resource->response == NULL )
+ {
+ msg_Err( p_access, "Failed to obtain a response from the request %s %s",
+ req_method.c_str(), current_url.c_str() );
+ vlc_http_res_destroy( resource );
+ on_completed( response_code, response, error_stream );
+ return;
+ }
+ response_code = vlc_http_msg_get_status( resource->response );
+
+ // Get the redirect URI before destroying the resource
+ char *redirect_uri = vlc_http_res_get_redirect( resource );
+ // Check for redirects if exist
+ if ( req_follow_redirect && redirect_uri != NULL &&
+ IHttpRequest::isRedirect( response_code ) )
+ {
+ redirect_counter += 1;
+ vlc_http_res_destroy( resource );
+ if ( redirect_counter > URL_REDIRECT_LIMIT )
+ {
+ msg_Err( p_access, "URL has been redirected too many times.");
+ on_completed( response_code, response, error_stream );
+ return;
+ }
+ current_url = std::string( redirect_uri );
+ goto request;
+ }
+
+ // Read the payload response into the buffer (ostream)
+ unsigned int content_length = 0;
+ struct block_t *block = vlc_http_res_read( resource );
+ while (block != NULL)
+ {
+ if ( IHttpRequest::isSuccess( response_code ) )
+ response->write( (char *) block->p_buffer, block->i_buffer );
+ else
+ error_stream->write( (char *) block->p_buffer, block->i_buffer );
+ content_length += block->i_buffer;
+
+ block_Release(block);
+ block = vlc_http_res_read( resource );
+ }
+
+ // Debug messages about the output
+ if ( IHttpRequest::isSuccess( response_code ) )
+ msg_Dbg( p_access, "%s %s succeed with a code %d", req_method.c_str(),
+ current_url.c_str(), response_code );
+ else
+ msg_Warn( p_access, "Failed to request %s %s with an error code of %d",
+ req_method.c_str(), current_url.c_str(), response_code );
+
+ // Invoke the respective callbacks
+ if ( cb != nullptr )
+ {
+ cb->receivedHttpCode( static_cast<int>( response_code ) );
+ cb->receivedContentLength( static_cast<int>( content_length ) );
+ }
+
+ vlc_http_res_destroy( resource );
+ on_completed( response_code, response, error_stream );
+}
+
+int HttpRequest::httpRequestHandler( const struct vlc_http_resource *res,
+ struct vlc_http_msg *req, void *opaque )
+{
+ (void) res;
+
+ // Inserting headers
+ HttpRequestData* data = static_cast<HttpRequestData *>( opaque );
+ for ( const auto& header : data->ptr->headerParameters() )
+ {
+ // Prevent duplicate entries and host header is controlled by
+ // http_file_create
+ if ( vlc_http_msg_get_header( req, header.first.c_str() ) == NULL &&
+ strcasecmp( header.first.c_str(), "Host" ) )
+ {
+ vlc_http_msg_add_header( req, header.first.c_str(), "%s",
+ header.second.c_str() );
+ }
+ }
+
+ // Pre-calculate the content-length of the stream (seekable)
+ std::streampos pos = data->data->tellg();
+ data->data->seekg( 0, std::ios::end );
+ std::streamsize content_length = data->data->tellg() - pos;
+ data->data->seekg( pos );
+
+ // Stream the data using VLC format (into block_t)
+ vlc_http_stream *stream = vlc_payload_stream_open( data->data );
+ vlc_http_msg_attach( req, stream );
+
+ // Content-Length is a mandatory field sometimes (e.g PUT methods)
+ if ( data->ptr->req_method != "GET" )
+ {
+ vlc_http_msg_add_header( req, "Content-Length", "%ld", content_length );
+ if ( vlc_http_msg_get_header( req, "Content-Type" ) == NULL &&
+ data->ptr->req_method != "PUT" && content_length > 0)
+ {
+ vlc_http_msg_add_header( req, "Content-Type", "%s",
+ "application/x-www-form-urlencoded" );
+ }
+ }
+ return VLC_SUCCESS;
+}
+
+int HttpRequest::httpResponseHandler( const struct vlc_http_resource *res,
+ const struct vlc_http_msg *resp, void *opaque )
+{
+ (void) res;
+ (void) resp;
+ (void) opaque;
+ return VLC_SUCCESS;
+}
+
+// Http interface implementation
+Http::Http( stream_t *access ) : p_access( access )
+{
+}
+
+cloudstorage::IHttpRequest::Pointer Http::create( const std::string& url,
+ const std::string& method, bool follow_redirect ) const
+{
+ return std::unique_ptr<HttpRequest>( new HttpRequest( p_access, url, method,
+ follow_redirect ) );
+}
diff --git a/modules/access/cloudstorage/provider_http.h b/modules/access/cloudstorage/provider_http.h
new file mode 100644
index 0000000000..1e93d4964f
--- /dev/null
+++ b/modules/access/cloudstorage/provider_http.h
@@ -0,0 +1,89 @@
+/*****************************************************************************
+ * provider_http.h: Inherit class IHttp from libcloudstorage
+ *****************************************************************************
+ * Copyright (C) 2017 VideoLabs and VideoLAN
+ *
+ * Authors: Diogo Silva <dbtdsilva at gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
+ *****************************************************************************/
+
+#ifndef VLC_CLOUDSTORAGE_PROVIDER_HTTP_H
+#define VLC_CLOUDSTORAGE_PROVIDER_HTTP_H
+
+#ifdef HAVE_CONFIG_H
+# include <config.h>
+#endif
+
+#include <IHttp.h>
+#include <string>
+#include <unordered_map>
+
+#include "access.h"
+
+class Http : public cloudstorage::IHttp
+{
+public:
+ Http( stream_t *access );
+ cloudstorage::IHttpRequest::Pointer create( const std::string&,
+ const std::string&, bool ) const override;
+
+private:
+ stream_t *p_access;
+};
+
+class HttpRequest : public cloudstorage::IHttpRequest
+{
+public:
+ HttpRequest( stream_t* access, const std::string& url,
+ const std::string& method, bool follow_redirect );
+ ~HttpRequest();
+
+ void setParameter( const std::string& parameter,
+ const std::string& value ) override;
+ void setHeaderParameter( const std::string& parameter,
+ const std::string& value ) override;
+
+ const std::unordered_map<std::string, std::string>& parameters()
+ const override;
+ const std::unordered_map<std::string, std::string>& headerParameters()
+ const override;
+
+ const std::string& url() const override;
+ const std::string& method() const override;
+ bool follow_redirect() const override;
+
+ void send( CompleteCallback on_completed,
+ std::shared_ptr<std::istream> data,
+ std::shared_ptr<std::ostream> response,
+ std::shared_ptr<std::ostream> error_stream = nullptr,
+ ICallback::Pointer = nullptr ) const override;
+
+private:
+ static int httpRequestHandler( const struct vlc_http_resource *res,
+ struct vlc_http_msg *req, void *opaque );
+ static int httpResponseHandler( const struct vlc_http_resource *res,
+ const struct vlc_http_msg *resp, void *opaque );
+
+ stream_t *p_access;
+ std::unordered_map<std::string, std::string> req_parameters;
+ std::unordered_map<std::string, std::string> req_header_parameters;
+ std::string req_url, req_method;
+ bool req_follow_redirect;
+ struct vlc_http_mgr *manager;
+ static const struct vlc_http_resource_cbs handler_callbacks;
+};
+
+#endif /* VLC_CLOUDSTORAGE_PROVIDER_HTTP_H */
diff --git a/modules/access/cloudstorage/provider_http_stream.cpp b/modules/access/cloudstorage/provider_http_stream.cpp
new file mode 100644
index 0000000000..75409f634d
--- /dev/null
+++ b/modules/access/cloudstorage/provider_http_stream.cpp
@@ -0,0 +1,96 @@
+/*****************************************************************************
+ * provider_http_stream.cpp: Stream for sending a payload to the request
+ *****************************************************************************
+ * Copyright (C) 2017 VideoLabs and VideoLAN
+ *
+ * Authors: Diogo Silva <dbtdsilva at gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
+ *****************************************************************************/
+
+#include "provider_http_stream.h"
+
+#ifdef HAVE_CONFIG_H
+# include <config.h>
+#endif
+
+#include <vlc_common.h>
+#include <vlc_block.h>
+
+#include <istream>
+
+extern "C" {
+#include "access/http/message.h"
+}
+
+struct vlc_payload_stream
+{
+ struct vlc_http_stream stream;
+ std::istream *payload_stream;
+};
+
+static struct vlc_http_msg *vlc_payload_wait( struct vlc_http_stream *stream )
+{
+ /* There cannot be headers during this stream. */
+ (void) stream;
+ return NULL;
+}
+
+static block_t *vlc_payload_read( struct vlc_http_stream *stream )
+{
+ struct vlc_payload_stream *s =
+ container_of( stream, struct vlc_payload_stream, stream );
+ block_t *block;
+
+ if ( s->payload_stream->eof() )
+ return NULL;
+
+ size_t size = 16384;
+ block = block_Alloc( size );
+ if ( unlikely( block == NULL ) )
+ return NULL;
+
+ s->payload_stream->read( (char *) block->p_buffer, size );
+ block->i_buffer = s->payload_stream->gcount();
+
+ return block;
+}
+
+static void vlc_payload_close( struct vlc_http_stream *stream, bool abort )
+{
+ struct vlc_payload_stream *s =
+ container_of( stream, struct vlc_payload_stream, stream );
+
+ (void) abort;
+ free( s );
+}
+
+static struct vlc_http_stream_cbs vlc_payload_callbacks =
+{
+ vlc_payload_wait,
+ vlc_payload_read,
+ vlc_payload_close,
+};
+
+struct vlc_http_stream *vlc_payload_stream_open( std::istream *payload_stream )
+{
+ struct vlc_payload_stream *s = (vlc_payload_stream *) malloc( sizeof (*s) );
+ if ( unlikely( s == NULL ) )
+ return NULL;
+
+ s->stream.cbs = &vlc_payload_callbacks;
+ s->payload_stream = payload_stream;
+ return &s->stream;
+}
diff --git a/modules/access/cloudstorage/provider_http_stream.h b/modules/access/cloudstorage/provider_http_stream.h
new file mode 100644
index 0000000000..55abbf1651
--- /dev/null
+++ b/modules/access/cloudstorage/provider_http_stream.h
@@ -0,0 +1,30 @@
+/*****************************************************************************
+ * provider_http_stream.h: Stream for sending a payload to the request
+ *****************************************************************************
+ * Copyright (C) 2017 VideoLabs and VideoLAN
+ *
+ * Authors: Diogo Silva <dbtdsilva at gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
+ *****************************************************************************/
+
+#ifndef VLC_CLOUDSTORAGE_PROVIDER_HTTP_STREAM_H
+#define VLC_CLOUDSTORAGE_PROVIDER_HTTP_STREAM_H
+
+#include <istream>
+
+struct vlc_http_stream *vlc_payload_stream_open( std::istream *payload_stream );
+
+#endif
\ No newline at end of file
diff --git a/po/POTFILES.in b/po/POTFILES.in
index dd1bbcaf74..bbcb8ca075 100644
--- a/po/POTFILES.in
+++ b/po/POTFILES.in
@@ -181,8 +181,12 @@ modules/access/cloudstorage.cpp
modules/access/cloudstorage/access.cpp
modules/access/cloudstorage/access.h
modules/access/cloudstorage/provider_callback.h
+modules/access/cloudstorage/provider_http.cpp
modules/access/cloudstorage/provider_httpd.cpp
modules/access/cloudstorage/provider_httpd.h
+modules/access/cloudstorage/provider_http.h
+modules/access/cloudstorage/provider_http_stream.cpp
+modules/access/cloudstorage/provider_http_stream.h
modules/access/cloudstorage/services_discovery.cpp
modules/access/cloudstorage/services_discovery.h
modules/access/concat.c
--
2.14.1
More information about the vlc-devel
mailing list