diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/Bindings/LuaState.cpp | 11 | ||||
| -rw-r--r-- | src/Bindings/LuaState.h | 24 | ||||
| -rw-r--r-- | src/Bindings/LuaTCPLink.cpp | 270 | ||||
| -rw-r--r-- | src/Bindings/LuaTCPLink.h | 54 | ||||
| -rw-r--r-- | src/Bindings/ManualBindings_Network.cpp | 334 | ||||
| -rw-r--r-- | src/HTTP/CMakeLists.txt | 2 | ||||
| -rw-r--r-- | src/HTTP/HTTPMessageParser.h | 3 | ||||
| -rw-r--r-- | src/HTTP/UrlClient.cpp | 675 | ||||
| -rw-r--r-- | src/HTTP/UrlClient.h | 152 | ||||
| -rw-r--r-- | src/OSSupport/Network.h | 33 | ||||
| -rw-r--r-- | src/OSSupport/TCPLinkImpl.cpp | 296 | ||||
| -rw-r--r-- | src/OSSupport/TCPLinkImpl.h | 75 |
12 files changed, 1627 insertions, 302 deletions
diff --git a/src/Bindings/LuaState.cpp b/src/Bindings/LuaState.cpp index ec6bdb48a..81893c946 100644 --- a/src/Bindings/LuaState.cpp +++ b/src/Bindings/LuaState.cpp @@ -2139,6 +2139,17 @@ cLuaState * cLuaState::QueryCanonLuaState(void) +void cLuaState::LogApiCallParamFailure(const char * a_FnName, const char * a_ParamNames) +{ + LOGWARNING("%s: Cannot read params: %s, bailing out.", a_FnName, a_ParamNames); + LogStackTrace(); + LogStackValues("Values on the stack"); +} + + + + + int cLuaState::ReportFnCallErrors(lua_State * a_LuaState) { LOGWARNING("LUA: %s", lua_tostring(a_LuaState, -1)); diff --git a/src/Bindings/LuaState.h b/src/Bindings/LuaState.h index cb68b9a98..32d346a19 100644 --- a/src/Bindings/LuaState.h +++ b/src/Bindings/LuaState.h @@ -301,6 +301,26 @@ public: return cLuaState(m_Ref.GetLuaState()).CallTableFn(m_Ref, a_FnName, std::forward<Args>(args)...); } + /** Calls the Lua function stored under the specified name in the referenced table, if still available. + A "self" parameter is injected in front of all the given parameters and is set to the callback table. + Returns true if callback has been called. + Returns false if the Lua state isn't valid anymore, or the function doesn't exist. */ + template <typename... Args> + bool CallTableFnWithSelf(const char * a_FnName, Args &&... args) + { + auto cs = m_CS; + if (cs == nullptr) + { + return false; + } + cCSLock Lock(*cs); + if (!m_Ref.IsValid()) + { + return false; + } + return cLuaState(m_Ref.GetLuaState()).CallTableFn(m_Ref, a_FnName, m_Ref, std::forward<Args>(args)...); + } + /** Set the contained reference to the table in the specified Lua state's stack position. If another table has been previously contained, it is unreferenced first. Returns true on success, false on failure (not a table at the specified stack pos). */ @@ -741,6 +761,10 @@ public: Returns nullptr if the canon Lua state cannot be queried. */ cLuaState * QueryCanonLuaState(void); + /** Outputs to log a warning about API call being unable to read its parameters from the stack, + logs the stack trace and stack values. */ + void LogApiCallParamFailure(const char * a_FnName, const char * a_ParamNames); + protected: cCriticalSection m_CS; diff --git a/src/Bindings/LuaTCPLink.cpp b/src/Bindings/LuaTCPLink.cpp index 466240a7d..905dfc5ac 100644 --- a/src/Bindings/LuaTCPLink.cpp +++ b/src/Bindings/LuaTCPLink.cpp @@ -6,6 +6,8 @@ #include "Globals.h" #include "LuaTCPLink.h" #include "LuaServerHandle.h" +#include "../PolarSSL++/X509Cert.h" +#include "../PolarSSL++/CryptoKey.h" @@ -48,13 +50,6 @@ cLuaTCPLink::~cLuaTCPLink() bool cLuaTCPLink::Send(const AString & a_Data) { - // If running in SSL mode, push the data into the SSL context instead: - if (m_SslContext != nullptr) - { - m_SslContext->Send(a_Data); - return true; - } - // Safely grab a copy of the link: auto link = m_Link; if (link == nullptr) @@ -144,12 +139,6 @@ void cLuaTCPLink::Shutdown(void) cTCPLinkPtr link = m_Link; if (link != nullptr) { - if (m_SslContext != nullptr) - { - m_SslContext->NotifyClose(); - m_SslContext->ResetSelf(); - m_SslContext.reset(); - } link->Shutdown(); } } @@ -164,12 +153,6 @@ void cLuaTCPLink::Close(void) cTCPLinkPtr link = m_Link; if (link != nullptr) { - if (m_SslContext != nullptr) - { - m_SslContext->NotifyClose(); - m_SslContext->ResetSelf(); - m_SslContext.reset(); - } link->Close(); } @@ -186,46 +169,31 @@ AString cLuaTCPLink::StartTLSClient( const AString & a_OwnPrivKeyPassword ) { - // Check preconditions: - if (m_SslContext != nullptr) - { - return "TLS is already active on this link"; - } - if ( - (a_OwnCertData.empty() && !a_OwnPrivKeyData.empty()) || - (!a_OwnCertData.empty() && a_OwnPrivKeyData.empty()) - ) - { - return "Either provide both the certificate and private key, or neither"; - } - - // Create the SSL context: - m_SslContext.reset(new cLinkSslContext(*this)); - m_SslContext->Initialize(true); - - // Create the peer cert, if required: - if (!a_OwnCertData.empty() && !a_OwnPrivKeyData.empty()) + auto link = m_Link; + if (link != nullptr) { - auto OwnCert = std::make_shared<cX509Cert>(); - int res = OwnCert->Parse(a_OwnCertData.data(), a_OwnCertData.size()); - if (res != 0) + cX509CertPtr ownCert; + if (!a_OwnCertData.empty()) { - m_SslContext.reset(); - return Printf("Cannot parse peer certificate: -0x%x", res); + ownCert = std::make_shared<cX509Cert>(); + auto res = ownCert->Parse(a_OwnCertData.data(), a_OwnCertData.size()); + if (res != 0) + { + return Printf("Cannot parse client certificate: -0x%x", res); + } } - auto OwnPrivKey = std::make_shared<cCryptoKey>(); - res = OwnPrivKey->ParsePrivate(a_OwnPrivKeyData.data(), a_OwnPrivKeyData.size(), a_OwnPrivKeyPassword); - if (res != 0) + cCryptoKeyPtr ownPrivKey; + if (!a_OwnPrivKeyData.empty()) { - m_SslContext.reset(); - return Printf("Cannot parse peer private key: -0x%x", res); + ownPrivKey = std::make_shared<cCryptoKey>(); + auto res = ownPrivKey->ParsePrivate(a_OwnPrivKeyData.data(), a_OwnPrivKeyData.size(), a_OwnPrivKeyPassword); + if (res != 0) + { + return Printf("Cannot parse client private key: -0x%x", res); + } } - m_SslContext->SetOwnCert(OwnCert, OwnPrivKey); + return link->StartTLSClient(ownCert, ownPrivKey); } - m_SslContext->SetSelf(cLinkSslContextWPtr(m_SslContext)); - - // Start the handshake: - m_SslContext->Handshake(); return ""; } @@ -240,43 +208,25 @@ AString cLuaTCPLink::StartTLSServer( const AString & a_StartTLSData ) { - // Check preconditions: - if (m_SslContext != nullptr) - { - return "TLS is already active on this link"; - } - if (a_OwnCertData.empty() || a_OwnPrivKeyData.empty()) + auto link = m_Link; + if (link != nullptr) { - return "Provide the server certificate and private key"; - } - - // Create the SSL context: - m_SslContext.reset(new cLinkSslContext(*this)); - m_SslContext->Initialize(false); - // Create the peer cert: auto OwnCert = std::make_shared<cX509Cert>(); int res = OwnCert->Parse(a_OwnCertData.data(), a_OwnCertData.size()); if (res != 0) { - m_SslContext.reset(); return Printf("Cannot parse server certificate: -0x%x", res); } auto OwnPrivKey = std::make_shared<cCryptoKey>(); res = OwnPrivKey->ParsePrivate(a_OwnPrivKeyData.data(), a_OwnPrivKeyData.size(), a_OwnPrivKeyPassword); if (res != 0) { - m_SslContext.reset(); return Printf("Cannot parse server private key: -0x%x", res); } - m_SslContext->SetOwnCert(OwnCert, OwnPrivKey); - m_SslContext->SetSelf(cLinkSslContextWPtr(m_SslContext)); - // Push the initial data: - m_SslContext->StoreReceivedData(a_StartTLSData.data(), a_StartTLSData.size()); - - // Start the handshake: - m_SslContext->Handshake(); + return link->StartTLSServer(OwnCert, OwnPrivKey, a_StartTLSData); + } return ""; } @@ -308,9 +258,6 @@ void cLuaTCPLink::Terminated(void) m_Link.reset(); } } - - // If the SSL context still exists, free it: - m_SslContext.reset(); } @@ -362,14 +309,6 @@ void cLuaTCPLink::OnLinkCreated(cTCPLinkPtr a_Link) void cLuaTCPLink::OnReceivedData(const char * a_Data, size_t a_Length) { - // If we're running in SSL mode, put the data into the SSL decryptor: - auto sslContext = m_SslContext; - if (sslContext != nullptr) - { - sslContext->StoreReceivedData(a_Data, a_Length); - return; - } - // Call the callback: m_Callbacks->CallTableFn("OnReceivedData", this, AString(a_Data, a_Length)); } @@ -380,13 +319,6 @@ void cLuaTCPLink::OnReceivedData(const char * a_Data, size_t a_Length) void cLuaTCPLink::OnRemoteClosed(void) { - // If running in SSL mode and there's data left in the SSL contect, report it: - auto sslContext = m_SslContext; - if (sslContext != nullptr) - { - sslContext->FlushBuffers(); - } - // Call the callback: m_Callbacks->CallTableFn("OnRemoteClosed", this); @@ -398,155 +330,3 @@ void cLuaTCPLink::OnRemoteClosed(void) -//////////////////////////////////////////////////////////////////////////////// -// cLuaTCPLink::cLinkSslContext: - -cLuaTCPLink::cLinkSslContext::cLinkSslContext(cLuaTCPLink & a_Link): - m_Link(a_Link) -{ -} - - - - - -void cLuaTCPLink::cLinkSslContext::SetSelf(cLinkSslContextWPtr a_Self) -{ - m_Self = a_Self; -} - - - - - -void cLuaTCPLink::cLinkSslContext::ResetSelf(void) -{ - m_Self.reset(); -} - - - - - -void cLuaTCPLink::cLinkSslContext::StoreReceivedData(const char * a_Data, size_t a_NumBytes) -{ - // Hold self alive for the duration of this function - cLinkSslContextPtr Self(m_Self); - - m_EncryptedData.append(a_Data, a_NumBytes); - - // Try to finish a pending handshake: - TryFinishHandshaking(); - - // Flush any cleartext data that can be "received": - FlushBuffers(); -} - - - - - -void cLuaTCPLink::cLinkSslContext::FlushBuffers(void) -{ - // Hold self alive for the duration of this function - cLinkSslContextPtr Self(m_Self); - - // If the handshake didn't complete yet, bail out: - if (!HasHandshaken()) - { - return; - } - - char Buffer[1024]; - int NumBytes; - while ((NumBytes = ReadPlain(Buffer, sizeof(Buffer))) > 0) - { - m_Link.ReceivedCleartextData(Buffer, static_cast<size_t>(NumBytes)); - if (m_Self.expired()) - { - // The callback closed the SSL context, bail out - return; - } - } -} - - - - - -void cLuaTCPLink::cLinkSslContext::TryFinishHandshaking(void) -{ - // Hold self alive for the duration of this function - cLinkSslContextPtr Self(m_Self); - - // If the handshake hasn't finished yet, retry: - if (!HasHandshaken()) - { - Handshake(); - } - - // If the handshake succeeded, write all the queued plaintext data: - if (HasHandshaken()) - { - WritePlain(m_CleartextData.data(), m_CleartextData.size()); - m_CleartextData.clear(); - } -} - - - - - -void cLuaTCPLink::cLinkSslContext::Send(const AString & a_Data) -{ - // Hold self alive for the duration of this function - cLinkSslContextPtr Self(m_Self); - - // If the handshake hasn't completed yet, queue the data: - if (!HasHandshaken()) - { - m_CleartextData.append(a_Data); - TryFinishHandshaking(); - return; - } - - // The connection is all set up, write the cleartext data into the SSL context: - WritePlain(a_Data.data(), a_Data.size()); - FlushBuffers(); -} - - - - - -int cLuaTCPLink::cLinkSslContext::ReceiveEncrypted(unsigned char * a_Buffer, size_t a_NumBytes) -{ - // Hold self alive for the duration of this function - cLinkSslContextPtr Self(m_Self); - - // If there's nothing queued in the buffer, report empty buffer: - if (m_EncryptedData.empty()) - { - return POLARSSL_ERR_NET_WANT_READ; - } - - // Copy as much data as possible to the provided buffer: - size_t BytesToCopy = std::min(a_NumBytes, m_EncryptedData.size()); - memcpy(a_Buffer, m_EncryptedData.data(), BytesToCopy); - m_EncryptedData.erase(0, BytesToCopy); - return static_cast<int>(BytesToCopy); -} - - - - - -int cLuaTCPLink::cLinkSslContext::SendEncrypted(const unsigned char * a_Buffer, size_t a_NumBytes) -{ - m_Link.m_Link->Send(a_Buffer, a_NumBytes); - return static_cast<int>(a_NumBytes); -} - - - - diff --git a/src/Bindings/LuaTCPLink.h b/src/Bindings/LuaTCPLink.h index b8c886ef8..f4ca67018 100644 --- a/src/Bindings/LuaTCPLink.h +++ b/src/Bindings/LuaTCPLink.h @@ -10,7 +10,6 @@ #pragma once #include "../OSSupport/Network.h" -#include "../PolarSSL++/SslContext.h" #include "LuaState.h" @@ -90,54 +89,6 @@ public: ); protected: - // fwd: - class cLinkSslContext; - typedef SharedPtr<cLinkSslContext> cLinkSslContextPtr; - typedef WeakPtr<cLinkSslContext> cLinkSslContextWPtr; - - /** Wrapper around cSslContext that is used when this link is being encrypted by SSL. */ - class cLinkSslContext : - public cSslContext - { - cLuaTCPLink & m_Link; - - /** Buffer for storing the incoming encrypted data until it is requested by the SSL decryptor. */ - AString m_EncryptedData; - - /** Buffer for storing the outgoing cleartext data until the link has finished handshaking. */ - AString m_CleartextData; - - /** Shared ownership of self, so that this object can keep itself alive for as long as it needs. */ - cLinkSslContextWPtr m_Self; - - public: - cLinkSslContext(cLuaTCPLink & a_Link); - - /** Shares ownership of self, so that this object can keep itself alive for as long as it needs. */ - void SetSelf(cLinkSslContextWPtr a_Self); - - /** Removes the self ownership so that we can detect the SSL closure. */ - void ResetSelf(void); - - /** Stores the specified block of data into the buffer of the data to be decrypted (incoming from remote). - Also flushes the SSL buffers by attempting to read any data through the SSL context. */ - void StoreReceivedData(const char * a_Data, size_t a_NumBytes); - - /** Tries to read any cleartext data available through the SSL, reports it in the link. */ - void FlushBuffers(void); - - /** Tries to finish handshaking the SSL. */ - void TryFinishHandshaking(void); - - /** Sends the specified cleartext data over the SSL to the remote peer. - If the handshake hasn't been completed yet, queues the data for sending when it completes. */ - void Send(const AString & a_Data); - - // cSslContext overrides: - virtual int ReceiveEncrypted(unsigned char * a_Buffer, size_t a_NumBytes) override; - virtual int SendEncrypted(const unsigned char * a_Buffer, size_t a_NumBytes) override; - }; - /** The Lua table that holds the callbacks to be invoked. */ cLuaState::cTableRefPtr m_Callbacks; @@ -149,11 +100,6 @@ protected: /** The server that is responsible for this link, if any. */ cLuaServerHandleWPtr m_Server; - /** The SSL context used for encryption, if this link uses SSL. - If valid, the link uses encryption through this context. */ - cLinkSslContextPtr m_SslContext; - - /** Common code called when the link is considered as terminated. Releases m_Link, m_Callbacks and this from m_Server, each when applicable. */ void Terminated(void); diff --git a/src/Bindings/ManualBindings_Network.cpp b/src/Bindings/ManualBindings_Network.cpp index c8565d23d..f59860938 100644 --- a/src/Bindings/ManualBindings_Network.cpp +++ b/src/Bindings/ManualBindings_Network.cpp @@ -2,6 +2,7 @@ // ManualBindings_Network.cpp // Implements the cNetwork-related API bindings for Lua +// Also implements the cUrlClient bindings #include "Globals.h" #include "LuaTCPLink.h" @@ -12,6 +13,7 @@ #include "LuaNameLookup.h" #include "LuaServerHandle.h" #include "LuaUDPEndpoint.h" +#include "../HTTP/UrlClient.h" @@ -903,6 +905,329 @@ static int tolua_cUDPEndpoint_Send(lua_State * L) +/** Used when the cUrlClient Lua request wants all the callbacks. +Maps each callback onto a Lua function callback in the callback table. */ +class cFullUrlClientCallbacks: + public cUrlClient::cCallbacks +{ +public: + /** Creates a new instance bound to the specified table of callbacks. */ + cFullUrlClientCallbacks(cLuaState::cTableRefPtr && a_Callbacks): + m_Callbacks(std::move(a_Callbacks)) + { + } + + + // cUrlClient::cCallbacks overrides: + virtual void OnConnected(cTCPLink & a_Link) override + { + // TODO: Cannot push a cTCPLink to Lua, need to translate via cLuaTCPLink + m_Callbacks->CallTableFnWithSelf("OnConnected", cLuaState::Nil, a_Link.GetRemoteIP(), a_Link.GetRemotePort()); + } + + + virtual bool OnCertificateReceived() override + { + // TODO: The received cert needs proper type specification from the underlying cUrlClient framework and in the Lua engine as well + bool res = true; + m_Callbacks->CallTableFnWithSelf("OnCertificateReceived", cLuaState::Return, res); + return res; + } + + + virtual void OnTlsHandshakeCompleted() override + { + m_Callbacks->CallTableFnWithSelf("OnTlsHandshakeCompleted"); + } + + + virtual void OnRequestSent() override + { + m_Callbacks->CallTableFnWithSelf("OnRequestSent"); + } + + + virtual void OnStatusLine(const AString & a_HttpVersion, int a_StatusCode, const AString & a_Rest) override + { + m_Callbacks->CallTableFnWithSelf("OnStatusLine", a_HttpVersion, a_StatusCode, a_Rest); + } + + + virtual void OnHeader(const AString & a_Key, const AString & a_Value) override + { + m_Callbacks->CallTableFnWithSelf("OnHeader", a_Key, a_Value); + m_Headers[a_Key] = a_Value; + } + + + virtual void OnHeadersFinished() override + { + m_Callbacks->CallTableFnWithSelf("OnHeadersFinished", m_Headers); + } + + + virtual void OnBodyData(const void * a_Data, size_t a_Size) override + { + m_Callbacks->CallTableFnWithSelf("OnBodyData", AString(reinterpret_cast<const char *>(a_Data), a_Size)); + } + + + virtual void OnBodyFinished() override + { + m_Callbacks->CallTableFnWithSelf("OnBodyFinished"); + } + + + virtual void OnError(const AString & a_ErrorMsg) override + { + m_Callbacks->CallTableFnWithSelf("OnError", a_ErrorMsg); + } + + + virtual void OnRedirecting(const AString & a_NewLocation) override + { + m_Callbacks->CallTableFnWithSelf("OnRedirecting", a_NewLocation); + } + + +protected: + /** The Lua table containing the callbacks. */ + cLuaState::cTableRefPtr m_Callbacks; + + /** Accumulator for all the headers to be reported in the OnHeadersFinished() callback. */ + AStringMap m_Headers; +}; + + + + + +/** Used when the cUrlClient Lua request has just a single callback. +The callback is used to report the entire body at once, together with the HTTP headers, or to report an error: +callback("BodyContents", {headers}) +callback(nil, "ErrorMessage") +Accumulates the body contents into a single string until the body is finished. +Accumulates all HTTP headers into an AStringMap. */ +class cSimpleUrlClientCallbacks: + public cUrlClient::cCallbacks +{ +public: + /** Creates a new instance that uses the specified callback to report when request finishes. */ + cSimpleUrlClientCallbacks(cLuaState::cCallbackPtr && a_Callback): + m_Callback(std::move(a_Callback)) + { + } + + + virtual void OnHeader(const AString & a_Key, const AString & a_Value) override + { + m_Headers[a_Key] = a_Value; + } + + + virtual void OnBodyData(const void * a_Data, size_t a_Size) override + { + m_ResponseBody.append(reinterpret_cast<const char *>(a_Data), a_Size); + } + + + virtual void OnBodyFinished() override + { + m_Callback->Call(m_ResponseBody, m_Headers); + } + + + virtual void OnError(const AString & a_ErrorMsg) override + { + m_Callback->Call(cLuaState::Nil, a_ErrorMsg); + } + + +protected: + + /** The callback to call when the request finishes. */ + cLuaState::cCallbackPtr m_Callback; + + /** The accumulator for the partial body data, so that OnBodyFinished() can send the entire thing at once. */ + AString m_ResponseBody; + + /** Accumulator for all the headers to be reported in the combined callback. */ + AStringMap m_Headers; +}; + + + + + +/** Common code shared among the cUrlClient request methods. +a_Method is the method name to be used in the request. +a_UrlStackIdx is the position on the Lua stack of the Url parameter. */ +static int tolua_cUrlClient_Request_Common(lua_State * a_LuaState, const AString & a_Method, int a_UrlStackIdx) +{ + // Check params: + cLuaState L(a_LuaState); + if (!L.CheckParamString(a_UrlStackIdx)) + { + return 0; + } + + // Read params: + AString url, requestBody; + AStringMap headers, options; + cLuaState::cTableRefPtr callbacks; + cLuaState::cCallbackPtr onCompleteBodyCallback; + if (!L.GetStackValues(a_UrlStackIdx, url)) + { + L.LogApiCallParamFailure("cUrlClient:Request()", Printf("URL (%d)", a_UrlStackIdx).c_str()); + L.Push(false); + L.Push("Invalid params"); + return 2; + } + cUrlClient::cCallbacksPtr urlClientCallbacks; + if (lua_istable(L, a_UrlStackIdx + 1)) + { + if (!L.GetStackValue(a_UrlStackIdx + 1, callbacks)) + { + L.LogApiCallParamFailure("cUrlClient:Request()", Printf("CallbacksTable (%d)", a_UrlStackIdx + 1).c_str()); + L.Push(false); + L.Push("Invalid Callbacks param"); + return 2; + } + urlClientCallbacks = cpp14::make_unique<cFullUrlClientCallbacks>(std::move(callbacks)); + } + else if (lua_isfunction(L, a_UrlStackIdx + 1)) + { + if (!L.GetStackValue(a_UrlStackIdx + 1, onCompleteBodyCallback)) + { + L.LogApiCallParamFailure("cUrlClient:Request()", Printf("CallbacksFn (%d)", a_UrlStackIdx + 1).c_str()); + L.Push(false); + L.Push("Invalid OnCompleteBodyCallback param"); + return 2; + } + urlClientCallbacks = cpp14::make_unique<cSimpleUrlClientCallbacks>(std::move(onCompleteBodyCallback)); + } + else + { + L.LogApiCallParamFailure("cUrlClient:Request()", Printf("Callbacks (%d)", a_UrlStackIdx + 1).c_str()); + L.Push(false); + L.Push("Invalid OnCompleteBodyCallback param"); + return 2; + } + if (!L.GetStackValues(a_UrlStackIdx + 2, cLuaState::cOptionalParam<AStringMap>(headers), cLuaState::cOptionalParam<AString>(requestBody), cLuaState::cOptionalParam<AStringMap>(options))) + { + L.LogApiCallParamFailure("cUrlClient:Request()", Printf("Header, Body or Options (%d, %d, %d)", a_UrlStackIdx + 2, a_UrlStackIdx + 3, a_UrlStackIdx + 4).c_str()); + L.Push(false); + L.Push("Invalid params"); + return 2; + } + + // Make the request: + auto res = cUrlClient::Request(a_Method, url, std::move(urlClientCallbacks), std::move(headers), std::move(requestBody), std::move(options)); + if (!res.first) + { + L.Push(false); + L.Push(res.second); + return 2; + } + L.Push(true); + return 1; +} + + + + + +/** Implements cUrlClient:Get() using cUrlClient::Request(). */ +static int tolua_cUrlClient_Delete(lua_State * a_LuaState) +{ + /* Function signatures: + cUrlClient:Delete(URL, {CallbacksFnTable}, [{HeadersMapTable}], [RequestBody], [{OptionsMapTable}]) -> true / false + string + cUrlClient:Delete(URL, OnCompleteBodyCallback, [{HeadersMapTable}], [RequestBody], [{OptionsMapTable}]) -> true / false + string + */ + + return tolua_cUrlClient_Request_Common(a_LuaState, "DELETE", 2); +} + + + + + +/** Implements cUrlClient:Get() using cUrlClient::Request(). */ +static int tolua_cUrlClient_Get(lua_State * a_LuaState) +{ + /* Function signatures: + cUrlClient:Get(URL, {CallbacksFnTable}, [{HeadersMapTable}], [RequestBody], [{OptionsMapTable}]) -> true / false + string + cUrlClient:Get(URL, OnCompleteBodyCallback, [{HeadersMapTable}], [RequestBody], [{OptionsMapTable}]) -> true / false + string + */ + + return tolua_cUrlClient_Request_Common(a_LuaState, "GET", 2); +} + + + + + +/** Implements cUrlClient:Post() using cUrlClient::Request(). */ +static int tolua_cUrlClient_Post(lua_State * a_LuaState) +{ + /* Function signatures: + cUrlClient:Post(URL, {CallbacksFnTable}, [{HeadersMapTable}], [RequestBody], [{OptionsMapTable}]) -> true / false + string + cUrlClient:Post(URL, OnCompleteBodyCallback, [{HeadersMapTable}], [RequestBody], [{OptionsMapTable}]) -> true / false + string + */ + + return tolua_cUrlClient_Request_Common(a_LuaState, "POST", 2); +} + + + + + +/** Implements cUrlClient:Put() using cUrlClient::Request(). */ +static int tolua_cUrlClient_Put(lua_State * a_LuaState) +{ + /* Function signatures: + cUrlClient:Put(URL, {CallbacksFnTable}, [{HeadersMapTable}], [RequestBody], [{OptionsMapTable}]) -> true / false + string + cUrlClient:Put(URL, OnCompleteBodyCallback, [{HeadersMapTable}], [RequestBody], [{OptionsMapTable}]) -> true / false + string + */ + + return tolua_cUrlClient_Request_Common(a_LuaState, "PUT", 2); +} + + + + + +/** Binds cUrlClient::Request(). */ +static int tolua_cUrlClient_Request(lua_State * a_LuaState) +{ + /* Function signatures: + cUrlClient:Request(Method, URL, {CallbacksFnTable}, [{HeadersMapTable}], [RequestBody], [{OptionsMapTable}]) -> true / false + string + cUrlClient:Request(Method, URL, OnCompleteBodyCallback, [{HeadersMapTable}], [RequestBody], [{OptionsMapTable}]) -> true / false + string + */ + + // Check that the Method param is a string: + cLuaState L(a_LuaState); + if (!L.CheckParamString(2)) + { + return 0; + } + + // Redirect the rest to the common code: + AString method; + if (!L.GetStackValue(2, method)) + { + L.LogApiCallParamFailure("cUrlClient:Request", "Method (2)"); + L.Push(cLuaState::Nil); + L.Push("Invalid params"); + return 2; + } + return tolua_cUrlClient_Request_Common(a_LuaState, method, 3); +} + + + + + //////////////////////////////////////////////////////////////////////////////// // Register the bindings: @@ -913,10 +1238,12 @@ void cManualBindings::BindNetwork(lua_State * tolua_S) tolua_usertype(tolua_S, "cServerHandle"); tolua_usertype(tolua_S, "cTCPLink"); tolua_usertype(tolua_S, "cUDPEndpoint"); + tolua_usertype(tolua_S, "cUrlClient"); tolua_cclass(tolua_S, "cNetwork", "cNetwork", "", nullptr); tolua_cclass(tolua_S, "cServerHandle", "cServerHandle", "", tolua_collect_cServerHandle); tolua_cclass(tolua_S, "cTCPLink", "cTCPLink", "", nullptr); tolua_cclass(tolua_S, "cUDPEndpoint", "cUDPEndpoint", "", tolua_collect_cUDPEndpoint); + tolua_cclass(tolua_S, "cUrlClient", "cUrlClient", "", nullptr); // Fill in the functions (alpha-sorted): tolua_beginmodule(tolua_S, "cNetwork"); @@ -953,6 +1280,13 @@ void cManualBindings::BindNetwork(lua_State * tolua_S) tolua_function(tolua_S, "Send", tolua_cUDPEndpoint_Send); tolua_endmodule(tolua_S); + tolua_beginmodule(tolua_S, "cUrlClient"); + tolua_function(tolua_S, "Delete", tolua_cUrlClient_Delete); + tolua_function(tolua_S, "Get", tolua_cUrlClient_Get); + tolua_function(tolua_S, "Post", tolua_cUrlClient_Post); + tolua_function(tolua_S, "Put", tolua_cUrlClient_Put); + tolua_function(tolua_S, "Request", tolua_cUrlClient_Request); + tolua_endmodule(tolua_S); } diff --git a/src/HTTP/CMakeLists.txt b/src/HTTP/CMakeLists.txt index 03cda2adc..3a2001e67 100644 --- a/src/HTTP/CMakeLists.txt +++ b/src/HTTP/CMakeLists.txt @@ -13,6 +13,7 @@ SET (SRCS NameValueParser.cpp SslHTTPServerConnection.cpp TransferEncodingParser.cpp + UrlClient.cpp UrlParser.cpp ) @@ -27,6 +28,7 @@ SET (HDRS NameValueParser.h SslHTTPServerConnection.h TransferEncodingParser.h + UrlClient.h UrlParser.h ) diff --git a/src/HTTP/HTTPMessageParser.h b/src/HTTP/HTTPMessageParser.h index f07de0492..307714243 100644 --- a/src/HTTP/HTTPMessageParser.h +++ b/src/HTTP/HTTPMessageParser.h @@ -31,7 +31,8 @@ public: /** Called when an error has occured while parsing. */ virtual void OnError(const AString & a_ErrorDescription) = 0; - /** Called when the first line (request / status) is fully parsed. */ + /** Called when the first line of the request or response is fully parsed. + Doesn't check the validity of the line, only extracts the first complete line. */ virtual void OnFirstLine(const AString & a_FirstLine) = 0; /** Called when a single header line is parsed. */ diff --git a/src/HTTP/UrlClient.cpp b/src/HTTP/UrlClient.cpp new file mode 100644 index 000000000..29d5e28d7 --- /dev/null +++ b/src/HTTP/UrlClient.cpp @@ -0,0 +1,675 @@ + +// UrlClient.cpp + +// Implements the cUrlClient class for high-level URL interaction + +#include "Globals.h" +#include "UrlClient.h" +#include "UrlParser.h" +#include "HTTPMessageParser.h" +#include "../PolarSSL++/X509Cert.h" +#include "../PolarSSL++/CryptoKey.h" + + + + + +// fwd: +class cSchemeHandler; +typedef SharedPtr<cSchemeHandler> cSchemeHandlerPtr; + + + + + +class cUrlClientRequest: + public cNetwork::cConnectCallbacks, + public cTCPLink::cCallbacks +{ + friend class cHttpSchemeHandler; + +public: + static std::pair<bool, AString> Request( + const AString & a_Method, + const AString & a_URL, + cUrlClient::cCallbacksPtr && a_Callbacks, + AStringMap && a_Headers, + const AString & a_Body, + AStringMap && a_Options + ) + { + // Create a new instance of cUrlClientRequest, wrapped in a SharedPtr, so that it has a controlled lifetime. + // Cannot use std::make_shared, because the constructor is not public + SharedPtr<cUrlClientRequest> ptr (new cUrlClientRequest( + a_Method, a_URL, std::move(a_Callbacks), std::move(a_Headers), a_Body, std::move(a_Options) + )); + return ptr->DoRequest(ptr); + } + + + /** Calls the error callback with the specified message, if it exists, and terminates the request. */ + void CallErrorCallback(const AString & a_ErrorMessage) + { + // Call the error callback: + m_Callbacks->OnError(a_ErrorMessage); + + // Terminate the request's TCP link: + auto link = m_Link; + if (link != nullptr) + { + link->Close(); + } + m_Self.reset(); + } + + + cUrlClient::cCallbacks & GetCallbacks() { return *m_Callbacks; } + + void RedirectTo(const AString & a_RedirectUrl); + + bool ShouldAllowRedirects() const; + + cX509CertPtr GetOwnCert() const + { + auto itr = m_Options.find("OwnCert"); + if (itr == m_Options.end()) + { + return nullptr; + } + cX509CertPtr cert; + if (!cert->Parse(itr->second.data(), itr->second.size())) + { + LOGD("OwnCert failed to parse"); + return nullptr; + } + return cert; + } + + cCryptoKeyPtr GetOwnPrivKey() const + { + auto itr = m_Options.find("OwnPrivKey"); + if (itr == m_Options.end()) + { + return nullptr; + } + cCryptoKeyPtr key; + auto passItr = m_Options.find("OwnPrivKeyPassword"); + auto pass = (passItr == m_Options.end()) ? AString() : passItr->second; + if (!key->ParsePrivate(itr->second.data(), itr->second.size(), pass)) + { + return nullptr; + } + return key; + } + +protected: + + /** Method to be used for the request */ + AString m_Method; + + /** URL that will be requested. */ + AString m_Url; + + /** Individual components of the URL that will be requested. */ + AString m_UrlScheme, m_UrlUsername, m_UrlPassword, m_UrlHost, m_UrlPath, m_UrlQuery, m_UrlFragment; + UInt16 m_UrlPort; + + /** Callbacks that report progress and results of the request. */ + cUrlClient::cCallbacksPtr m_Callbacks; + + /** Extra headers to be sent with the request (besides the normal ones). */ + AStringMap m_Headers; + + /** Body to be sent with the request, if any. */ + AString m_Body; + + /** Extra options to be used for the request. */ + AStringMap m_Options; + + /** SharedPtr to self, so that this object can keep itself alive for as long as it needs, + and pass self as callbacks to cNetwork functions. */ + SharedPtr<cUrlClientRequest> m_Self; + + /** The handler that "talks" the protocol specified in m_UrlScheme, handles all the sending and receiving. */ + SharedPtr<cSchemeHandler> m_SchemeHandler; + + /** The link handling the request. */ + cTCPLinkPtr m_Link; + + /** The number of redirect attempts that will still be followed. + If the response specifies a redirect and this is nonzero, the redirect is followed. + If the response specifies a redirect and this is zero, a redirect loop is reported as an error. */ + int m_NumRemainingRedirects; + + + cUrlClientRequest( + const AString & a_Method, + const AString & a_Url, + cUrlClient::cCallbacksPtr && a_Callbacks, + AStringMap && a_Headers, + const AString & a_Body, + AStringMap && a_Options + ): + m_Method(a_Method), + m_Url(a_Url), + m_Callbacks(std::move(a_Callbacks)), + m_Headers(std::move(a_Headers)), + m_Body(a_Body), + m_Options(std::move(a_Options)) + { + m_NumRemainingRedirects = GetStringMapInteger(m_Options, "MaxRedirects", 30); + } + + + std::pair<bool, AString> DoRequest(SharedPtr<cUrlClientRequest> a_Self); + + + // cNetwork::cConnectCallbacks override: TCP link connected: + virtual void OnConnected(cTCPLink & a_Link) override; + + // cNetwork::cConnectCallbacks override: An error has occurred: + virtual void OnError(int a_ErrorCode, const AString & a_ErrorMsg) override + { + m_Callbacks->OnError(Printf("Network error %d (%s)", a_ErrorCode, a_ErrorMsg.c_str())); + m_Self.reset(); + } + + + // cTCPLink::cCallbacks override: TCP link created + virtual void OnLinkCreated(cTCPLinkPtr a_Link) override + { + m_Link = a_Link; + } + + + // cTCPLink::cCallbacks override: TLS handshake completed on the link: + virtual void OnTlsHandshakeCompleted(void) override; + + /** Called when there's data incoming from the remote peer. */ + virtual void OnReceivedData(const char * a_Data, size_t a_Length) override; + + + /** Called when the remote end closes the connection. + The link is still available for connection information query (IP / port). + Sending data on the link is not an error, but the data won't be delivered. */ + virtual void OnRemoteClosed(void) override; +}; + + + + + +/** Represents a base class for an object that "talks" a specified URL protocol, such as HTTP or FTP. +Also provides a static factory method for creating an instance based on the scheme. +A descendant of this class is created for each request and handles all of the request's aspects, +from right after connecting to the TCP link till the link is closed. +For an example of a specific handler, see the cHttpSchemeHandler class. */ +class cSchemeHandler abstract +{ +public: + cSchemeHandler(cUrlClientRequest & a_ParentRequest): + m_ParentRequest(a_ParentRequest) + { + } + + // Force a virtual destructor in all descendants + virtual ~cSchemeHandler() {} + + /** Creates and returns a new handler for the specified scheme. + a_ParentRequest is the request which is to be handled by the handler. */ + static cSchemeHandlerPtr Create(const AString & a_Scheme, cUrlClientRequest & a_ParentRequest); + + /** Called when the link gets established. */ + virtual void OnConnected(cTCPLink & a_Link) = 0; + + /** Called when there's data incoming from the remote peer. */ + virtual void OnReceivedData(const char * a_Data, size_t a_Length) = 0; + + /** Called when the TLS handshake has completed on the underlying link. */ + virtual void OnTlsHandshakeCompleted(void) = 0; + + /** Called when the remote end closes the connection. + The link is still available for connection information query (IP / port). + Sending data on the link is not an error, but the data won't be delivered. */ + virtual void OnRemoteClosed(void) = 0; + +protected: + cUrlClientRequest & m_ParentRequest; +}; + + + + + +/** cSchemeHandler descendant that handles HTTP (and HTTPS) requests. */ +class cHttpSchemeHandler: + public cSchemeHandler, + protected cHTTPMessageParser::cCallbacks +{ + typedef cSchemeHandler Super; + +public: + cHttpSchemeHandler(cUrlClientRequest & a_ParentRequest, bool a_IsTls): + Super(a_ParentRequest), + m_Parser(*this), + m_IsTls(a_IsTls), + m_IsRedirect(false) + { + } + + + virtual void OnConnected(cTCPLink & a_Link) override + { + m_Link = &a_Link; + if (m_IsTls) + { + m_Link->StartTLSClient(m_ParentRequest.GetOwnCert(), m_ParentRequest.GetOwnPrivKey()); + } + else + { + SendRequest(); + } + } + + + /** Sends the HTTP request over the link. + Common code for both HTTP and HTTPS. */ + void SendRequest() + { + // Compose the request line: + auto requestLine = m_ParentRequest.m_UrlPath; + if (requestLine.empty()) + { + requestLine = "/"; + } + if (!m_ParentRequest.m_UrlQuery.empty()) + { + requestLine.push_back('?'); + requestLine.append(m_ParentRequest.m_UrlQuery); + } + m_Link->Send(Printf("%s %s HTTP/1.1\r\n", m_ParentRequest.m_Method.c_str(), requestLine.c_str())); + + // Send the headers: + m_Link->Send(Printf("Host: %s\r\n", m_ParentRequest.m_UrlHost.c_str())); + m_Link->Send(Printf("Content-Length: %u\r\n", static_cast<unsigned>(m_ParentRequest.m_Body.size()))); + for (auto itr = m_ParentRequest.m_Headers.cbegin(), end = m_ParentRequest.m_Headers.cend(); itr != end; ++itr) + { + m_Link->Send(Printf("%s: %s\r\n", itr->first.c_str(), itr->second.c_str())); + } // for itr - m_Headers[] + m_Link->Send("\r\n", 2); + + // Send the body: + m_Link->Send(m_ParentRequest.m_Body); + + // Notify the callbacks that the request has been sent: + m_ParentRequest.GetCallbacks().OnRequestSent(); + } + + + virtual void OnReceivedData(const char * a_Data, size_t a_Length) override + { + auto res = m_Parser.Parse(a_Data, a_Length); + if (res == AString::npos) + { + m_ParentRequest.CallErrorCallback("Failed to parse HTTP response"); + return; + } + } + + + virtual void OnTlsHandshakeCompleted(void) override + { + SendRequest(); + } + + + virtual void OnRemoteClosed(void) override + { + m_Link = nullptr; + } + + + // cHTTPResponseParser::cCallbacks overrides: + virtual void OnError(const AString & a_ErrorDescription) override + { + m_ParentRequest.CallErrorCallback(a_ErrorDescription); + m_Link = nullptr; + } + + + virtual void OnFirstLine(const AString & a_FirstLine) override + { + // Find the first space, parse the result code between it and the second space: + auto idxFirstSpace = a_FirstLine.find(' '); + if (idxFirstSpace == AString::npos) + { + m_ParentRequest.CallErrorCallback(Printf("Failed to parse HTTP status line \"%s\", no space delimiter.", a_FirstLine.c_str())); + return; + } + auto idxSecondSpace = a_FirstLine.find(' ', idxFirstSpace + 1); + if (idxSecondSpace == AString::npos) + { + m_ParentRequest.CallErrorCallback(Printf("Failed to parse HTTP status line \"%s\", missing second space delimiter.", a_FirstLine.c_str())); + return; + } + int resultCode; + auto resultCodeStr = a_FirstLine.substr(idxFirstSpace + 1, idxSecondSpace - idxFirstSpace - 1); + if (!StringToInteger(resultCodeStr, resultCode)) + { + m_ParentRequest.CallErrorCallback(Printf("Failed to parse HTTP result code from response \"%s\"", resultCodeStr.c_str())); + return; + } + + // Check for redirects, follow if allowed by the options: + switch (resultCode) + { + case cUrlClient::HTTP_STATUS_MULTIPLE_CHOICES: + case cUrlClient::HTTP_STATUS_MOVED_PERMANENTLY: + case cUrlClient::HTTP_STATUS_FOUND: + case cUrlClient::HTTP_STATUS_SEE_OTHER: + case cUrlClient::HTTP_STATUS_TEMPORARY_REDIRECT: + { + m_IsRedirect = true; + return; + } + } + m_ParentRequest.GetCallbacks().OnStatusLine(a_FirstLine.substr(1, idxFirstSpace), resultCode, a_FirstLine.substr(idxSecondSpace + 1)); + } + + + virtual void OnHeaderLine(const AString & a_Key, const AString & a_Value) override + { + if (m_IsRedirect) + { + if (a_Key == "Location") + { + m_RedirectLocation = a_Value; + } + } + else + { + m_ParentRequest.GetCallbacks().OnHeader(a_Key, a_Value); + } + } + + + /** Called when all the headers have been parsed. */ + virtual void OnHeadersFinished(void) override + { + if (!m_IsRedirect) + { + m_ParentRequest.GetCallbacks().OnHeadersFinished(); + } + } + + + /** Called for each chunk of the incoming body data. */ + virtual void OnBodyData(const void * a_Data, size_t a_Size) override + { + if (!m_IsRedirect) + { + m_ParentRequest.GetCallbacks().OnBodyData(a_Data, a_Size); + } + } + + + /** Called when the entire body has been reported by OnBodyData(). */ + virtual void OnBodyFinished(void) override + { + if (m_IsRedirect) + { + if (m_RedirectLocation.empty()) + { + m_ParentRequest.CallErrorCallback("Invalid redirect, there's no location to redirect to"); + } + else + { + m_ParentRequest.RedirectTo(m_RedirectLocation); + } + } + else + { + m_ParentRequest.GetCallbacks().OnBodyFinished(); + } + } + +protected: + + /** The network link. */ + cTCPLink * m_Link; + + /** Parser of the HTTP response message. */ + cHTTPMessageParser m_Parser; + + /** If true, the TLS should be started on the link before sending the request (used for https). */ + bool m_IsTls; + + /** Set to true if the first line contains a redirecting HTTP status code and the options specify to follow redirects. + If true, and the parent request allows redirects, neither headers not the body contents are reported through the callbacks, + and after the entire request is parsed, the redirect is attempted. */ + bool m_IsRedirect; + + /** The Location where the request should be redirected. + Only used when m_IsRedirect is true. */ + AString m_RedirectLocation; +}; + + + + + +//////////////////////////////////////////////////////////////////////////////// +// cSchemeHandler: + +cSchemeHandlerPtr cSchemeHandler::Create(const AString & a_Scheme, cUrlClientRequest & a_ParentRequest) +{ + auto lowerScheme = StrToLower(a_Scheme); + if (lowerScheme == "http") + { + return std::make_shared<cHttpSchemeHandler>(a_ParentRequest, false); + } + else if (lowerScheme == "https") + { + return std::make_shared<cHttpSchemeHandler>(a_ParentRequest, true); + } + + return nullptr; +} + + + + + +//////////////////////////////////////////////////////////////////////////////// +// cUrlClientRequest: + +void cUrlClientRequest::RedirectTo(const AString & a_RedirectUrl) +{ + // Check that redirection is allowed: + m_Callbacks->OnRedirecting(a_RedirectUrl); + if (!ShouldAllowRedirects()) + { + CallErrorCallback(Printf("Redirect to \"%s\" not allowed", a_RedirectUrl.c_str())); + return; + } + + // Do the actual redirect: + m_Link->Close(); + m_Url = a_RedirectUrl; + m_NumRemainingRedirects = m_NumRemainingRedirects - 1; + auto res = DoRequest(m_Self); + if (!res.first) + { + m_Callbacks->OnError(Printf("Redirection failed: %s", res.second.c_str())); + return; + } +} + + + + + +bool cUrlClientRequest::ShouldAllowRedirects() const +{ + return (m_NumRemainingRedirects > 0); +} + + + + + +void cUrlClientRequest::OnConnected(cTCPLink & a_Link) +{ + m_Callbacks->OnConnected(a_Link); + m_SchemeHandler->OnConnected(a_Link); +} + + + + + +void cUrlClientRequest::OnTlsHandshakeCompleted(void) +{ + // Notify the scheme handler and the callbacks: + m_SchemeHandler->OnTlsHandshakeCompleted(); + m_Callbacks->OnTlsHandshakeCompleted(); +} + + + + + +void cUrlClientRequest::OnReceivedData(const char * a_Data, size_t a_Length) +{ + auto handler = m_SchemeHandler; + if (handler != nullptr) + { + handler->OnReceivedData(a_Data, a_Length); + } +} + + + + + +void cUrlClientRequest::OnRemoteClosed() +{ + // Notify the callback: + auto handler = m_SchemeHandler; + if (handler != nullptr) + { + handler->OnRemoteClosed(); + } + + // Let ourselves be deleted + m_Self.reset(); +} + + + + + +std::pair<bool, AString> cUrlClientRequest::DoRequest(SharedPtr<cUrlClientRequest> a_Self) +{ + // We need a shared pointer to self, care must be taken not to pass any other ptr: + ASSERT(a_Self.get() == this); + + m_Self = a_Self; + + // Parse the URL: + auto res = cUrlParser::Parse(m_Url, m_UrlScheme, m_UrlUsername, m_UrlPassword, m_UrlHost, m_UrlPort, m_UrlPath, m_UrlQuery, m_UrlFragment); + if (!res.first) + { + return res; + } + + // Get a handler that will work with the specified scheme: + m_SchemeHandler = cSchemeHandler::Create(m_UrlScheme, *this); + if (m_SchemeHandler == nullptr) + { + return std::make_pair(false, Printf("Unknown Url scheme: %s", m_UrlScheme.c_str())); + } + + if (!cNetwork::Connect(m_UrlHost, m_UrlPort, m_Self, m_Self)) + { + return std::make_pair(false, "Network connection failed"); + } + return std::make_pair(true, AString()); +} + + + + + +//////////////////////////////////////////////////////////////////////////////// +// cUrlClient: + +std::pair<bool, AString> cUrlClient::Request( + const AString & a_Method, + const AString & a_URL, + cCallbacksPtr && a_Callbacks, + AStringMap && a_Headers, + AString && a_Body, + AStringMap && a_Options +) +{ + return cUrlClientRequest::Request( + a_Method, a_URL, std::move(a_Callbacks), std::move(a_Headers), std::move(a_Body), std::move(a_Options) + ); +} + + + + + +std::pair<bool, AString> cUrlClient::Get( + const AString & a_URL, + cCallbacksPtr && a_Callbacks, + AStringMap a_Headers, + AString a_Body, + AStringMap a_Options +) +{ + return cUrlClientRequest::Request( + "GET", a_URL, std::move(a_Callbacks), std::move(a_Headers), std::move(a_Body), std::move(a_Options) + ); +} + + + + + +std::pair<bool, AString> cUrlClient::Post( + const AString & a_URL, + cCallbacksPtr && a_Callbacks, + AStringMap && a_Headers, + AString && a_Body, + AStringMap && a_Options +) +{ + return cUrlClientRequest::Request( + "POST", a_URL, std::move(a_Callbacks), std::move(a_Headers), std::move(a_Body), std::move(a_Options) + ); +} + + + + + +std::pair<bool, AString> cUrlClient::Put( + const AString & a_URL, + cCallbacksPtr && a_Callbacks, + AStringMap && a_Headers, + AString && a_Body, + AStringMap && a_Options +) +{ + return cUrlClientRequest::Request( + "PUT", a_URL, std::move(a_Callbacks), std::move(a_Headers), std::move(a_Body), std::move(a_Options) + ); +} + + + + + diff --git a/src/HTTP/UrlClient.h b/src/HTTP/UrlClient.h new file mode 100644 index 000000000..ac772235e --- /dev/null +++ b/src/HTTP/UrlClient.h @@ -0,0 +1,152 @@ + +// UrlClient.h + +// Declares the cUrlClient class for high-level URL interaction + +/* +Options that can be set via the Options parameter to the cUrlClient calls: +"MaxRedirects": The maximum number of allowed redirects before the client refuses a redirect with an error +"OwnCert": The client certificate to use, if requested by the server. Any string that can be parsed by cX509Cert. +"OwnPrivKey": The private key appropriate for OwnCert. Any string that can be parsed by cCryptoKey. +"OwnPrivKeyPassword": The password for OwnPrivKey. If not present or empty, no password is assumed. + +Behavior: +- If a redirect is received, and redirection is allowed, the redirection is reported via OnRedirecting() callback +and the request is restarted at the redirect URL, without reporting any of the redirect's headers nor body +- If a redirect is received and redirection is not allowed (maximum redirection attempts have been reached), +the OnRedirecting() callback is called with the redirect URL and then the request terminates with an OnError() callback, +without reporting the redirect's headers nor body. +*/ + + + + + +#pragma once + +#include "../OSSupport/Network.h" + + + + + +class cUrlClient +{ +public: + /** Callbacks that are used for progress and result reporting. */ + class cCallbacks + { + public: + // Force a virtual destructor in descendants: + virtual ~cCallbacks() {} + + /** Called when the TCP connection is established. */ + virtual void OnConnected(cTCPLink & a_Link) {} + + /** Called for TLS connections, when the server certificate is received. + Return true to continue with the request, false to abort. + The default implementation does nothing and continues with the request. + TODO: The certificate parameter needs a representation! */ + virtual bool OnCertificateReceived() { return true; } + + /** Called for TLS connections, when the TLS handshake has been completed. + An empty default implementation is provided so that clients don't need to reimplement it unless they are interested in the event. */ + virtual void OnTlsHandshakeCompleted() { } + + /** Called after the entire request has been sent to the remote peer. */ + virtual void OnRequestSent() {} + + /** Called after the first line of the response is parsed, unless the response is an allowed redirect. */ + virtual void OnStatusLine(const AString & a_HttpVersion, int a_StatusCode, const AString & a_Rest) {} + + /** Called when a single HTTP header is received and parsed, unless the response is an allowed redirect + Called once for each incoming header. */ + virtual void OnHeader(const AString & a_Key, const AString & a_Value) {} + + /** Called when the HTTP headers have been fully parsed, unless the response is an allowed redirect. + There will be no more OnHeader() calls. */ + virtual void OnHeadersFinished() {} + + /** Called when the next fragment of the response body is received, unless the response is an allowed redirect. + This can be called multiple times, as data arrives over the network. */ + virtual void OnBodyData(const void * a_Data, size_t a_Size) {} + + /** Called after the response body has been fully reported by OnBody() calls, unless the response is an allowed redirect. + There will be no more OnBody() calls. */ + virtual void OnBodyFinished() {} + + /** Called when an asynchronous error is encountered. */ + virtual void OnError(const AString & a_ErrorMsg) {} + + /** Called when a redirect is to be followed. + This is called even if the redirecting is prohibited by the options; in such an event, this call will be + followed by OnError(). + If a response indicates a redirect (and the request allows redirecting), the regular callbacks + OnStatusLine(), OnHeader(), OnHeadersFinished(), OnBodyData() and OnBodyFinished() are not called + for such a response; instead, the redirect is silently attempted. */ + virtual void OnRedirecting(const AString & a_NewLocation) {} + }; + typedef UniquePtr<cCallbacks> cCallbacksPtr; + + + /** Used for HTTP status codes. */ + enum eHTTPStatus + { + HTTP_STATUS_OK = 200, + HTTP_STATUS_MULTIPLE_CHOICES = 300, // MAY have a redirect using the "Location" header + HTTP_STATUS_MOVED_PERMANENTLY = 301, // redirect using the "Location" header + HTTP_STATUS_FOUND = 302, // redirect using the "Location" header + HTTP_STATUS_SEE_OTHER = 303, // redirect using the "Location" header + HTTP_STATUS_TEMPORARY_REDIRECT = 307, // redirect using the "Location" header + }; + + + /** Makes a network request to the specified URL, using the specified method (if applicable). + The response is reported via the a_ResponseCallback callback, in a single call. + The metadata about the response (HTTP headers) are reported via a_InfoCallback before the a_ResponseCallback call. + If there is an asynchronous error, it is reported in via the a_ErrorCallback. + If there is an immediate error (misformatted URL etc.), the function returns false and an error message. + a_Headers contains additional headers to use for the request. + a_Body specifies optional body to include with the request, if applicable. + a_Options contains various options for the request that govern the request behavior, but aren't sent to the server, + such as the proxy server, whether to follow redirects, and client certificate for TLS. */ + static std::pair<bool, AString> Request( + const AString & a_Method, + const AString & a_URL, + cCallbacksPtr && a_Callbacks, + AStringMap && a_Headers, + AString && a_Body, + AStringMap && a_Options + ); + + /** Alias for Request("GET", ...) */ + static std::pair<bool, AString> Get( + const AString & a_URL, + cCallbacksPtr && a_Callbacks, + AStringMap a_Headers = AStringMap(), + AString a_Body = AString(), + AStringMap a_Options = AStringMap() + ); + + /** Alias for Request("POST", ...) */ + static std::pair<bool, AString> Post( + const AString & a_URL, + cCallbacksPtr && a_Callbacks, + AStringMap && a_Headers, + AString && a_Body, + AStringMap && a_Options + ); + + /** Alias for Request("PUT", ...) */ + static std::pair<bool, AString> Put( + const AString & a_URL, + cCallbacksPtr && a_Callbacks, + AStringMap && a_Headers, + AString && a_Body, + AStringMap && a_Options + ); +}; + + + + diff --git a/src/OSSupport/Network.h b/src/OSSupport/Network.h index 1162d7fc6..78c5e92f0 100644 --- a/src/OSSupport/Network.h +++ b/src/OSSupport/Network.h @@ -20,6 +20,11 @@ typedef std::vector<cTCPLinkPtr> cTCPLinkPtrs; class cServerHandle; typedef SharedPtr<cServerHandle> cServerHandlePtr; typedef std::vector<cServerHandlePtr> cServerHandlePtrs; +class cCryptoKey; +typedef SharedPtr<cCryptoKey> cCryptoKeyPtr; +class cX509Cert; +typedef SharedPtr<cX509Cert> cX509CertPtr; + @@ -49,6 +54,10 @@ public: Sending data on the link is not an error, but the data won't be delivered. */ virtual void OnRemoteClosed(void) = 0; + /** Called when the TLS handshake has been completed and communication can continue regularly. + Has an empty default implementation, so that link callback descendants don't need to specify TLS handlers when they don't use TLS at all. */ + virtual void OnTlsHandshakeCompleted(void) {} + /** Called when an error is detected on the connection. */ virtual void OnError(int a_ErrorCode, const AString & a_ErrorMsg) = 0; }; @@ -90,6 +99,30 @@ public: Sends the RST packet, queued outgoing and incoming data is lost. */ virtual void Close(void) = 0; + /** Starts a TLS handshake as a client connection. + If a client certificate should be used for the connection, set the certificate into a_OwnCertData and + its corresponding private key to a_OwnPrivKeyData. If both are empty, no client cert is presented. + a_OwnPrivKeyPassword is the password to be used for decoding PrivKey, empty if not passworded. + Returns empty string on success, non-empty error description on failure. */ + virtual AString StartTLSClient( + cX509CertPtr a_OwnCert, + cCryptoKeyPtr a_OwnPrivKey + ) = 0; + + /** Starts a TLS handshake as a server connection. + Set the server certificate into a_CertData and its corresponding private key to a_OwnPrivKeyData. + a_OwnPrivKeyPassword is the password to be used for decoding PrivKey, empty if not passworded. + a_StartTLSData is any data that should be pushed into the TLS before reading more data from the remote. + This is used mainly for protocols starting TLS in the middle of communication, when the TLS start command + can be received together with the TLS Client Hello message in one OnReceivedData() call, to re-queue the + Client Hello message into the TLS handshake buffer. + Returns empty string on success, non-empty error description on failure. */ + virtual AString StartTLSServer( + cX509CertPtr a_OwnCert, + cCryptoKeyPtr a_OwnPrivKey, + const AString & a_StartTLSData + ) = 0; + /** Returns the callbacks that are used. */ cCallbacksPtr GetCallbacks(void) const { return m_Callbacks; } diff --git a/src/OSSupport/TCPLinkImpl.cpp b/src/OSSupport/TCPLinkImpl.cpp index b15b6282f..d55dc9da1 100644 --- a/src/OSSupport/TCPLinkImpl.cpp +++ b/src/OSSupport/TCPLinkImpl.cpp @@ -48,6 +48,9 @@ cTCPLinkImpl::cTCPLinkImpl(evutil_socket_t a_Socket, cTCPLink::cCallbacksPtr a_L cTCPLinkImpl::~cTCPLinkImpl() { + // If the TLS context still exists, free it: + m_TlsContext.reset(); + bufferevent_free(m_BufferEvent); } @@ -129,7 +132,16 @@ bool cTCPLinkImpl::Send(const void * a_Data, size_t a_Length) LOGD("%s: Cannot send data, the link is already shut down.", __FUNCTION__); return false; } - return (bufferevent_write(m_BufferEvent, a_Data, a_Length) == 0); + + // If running in TLS mode, push the data into the TLS context instead: + if (m_TlsContext != nullptr) + { + m_TlsContext->Send(a_Data, a_Length); + return true; + } + + // Send the data: + return SendRaw(a_Data, a_Length); } @@ -138,6 +150,14 @@ bool cTCPLinkImpl::Send(const void * a_Data, size_t a_Length) void cTCPLinkImpl::Shutdown(void) { + // If running in TLS mode, notify the TLS layer: + if (m_TlsContext != nullptr) + { + m_TlsContext->NotifyClose(); + m_TlsContext->ResetSelf(); + m_TlsContext.reset(); + } + // If there's no outgoing data, shutdown the socket directly: if (evbuffer_get_length(bufferevent_get_output(m_BufferEvent)) == 0) { @@ -155,6 +175,14 @@ void cTCPLinkImpl::Shutdown(void) void cTCPLinkImpl::Close(void) { + // If running in TLS mode, notify the TLS layer: + if (m_TlsContext != nullptr) + { + m_TlsContext->NotifyClose(); + m_TlsContext->ResetSelf(); + m_TlsContext.reset(); + } + // Disable all events on the socket, but keep it alive: bufferevent_disable(m_BufferEvent, EV_READ | EV_WRITE); if (m_Server == nullptr) @@ -173,18 +201,98 @@ void cTCPLinkImpl::Close(void) +AString cTCPLinkImpl::StartTLSClient( + cX509CertPtr a_OwnCert, + cCryptoKeyPtr a_OwnPrivKey +) +{ + // Check preconditions: + if (m_TlsContext != nullptr) + { + return "TLS is already active on this link"; + } + if ( + ((a_OwnCert == nullptr) && (a_OwnPrivKey != nullptr)) || + ((a_OwnCert != nullptr) && (a_OwnPrivKey != nullptr)) + ) + { + return "Either provide both the certificate and private key, or neither"; + } + + // Create the TLS context: + m_TlsContext.reset(new cLinkTlsContext(*this)); + m_TlsContext->Initialize(true); + if (a_OwnCert != nullptr) + { + m_TlsContext->SetOwnCert(a_OwnCert, a_OwnPrivKey); + } + m_TlsContext->SetSelf(cLinkTlsContextWPtr(m_TlsContext)); + + // Start the handshake: + m_TlsContext->Handshake(); + return ""; +} + + + + + +AString cTCPLinkImpl::StartTLSServer( + cX509CertPtr a_OwnCert, + cCryptoKeyPtr a_OwnPrivKey, + const AString & a_StartTLSData +) +{ + // Check preconditions: + if (m_TlsContext != nullptr) + { + return "TLS is already active on this link"; + } + if ((a_OwnCert == nullptr) || (a_OwnPrivKey == nullptr)) + { + return "Provide the server certificate and private key"; + } + + // Create the TLS context: + m_TlsContext.reset(new cLinkTlsContext(*this)); + m_TlsContext->Initialize(false); + m_TlsContext->SetOwnCert(a_OwnCert, a_OwnPrivKey); + m_TlsContext->SetSelf(cLinkTlsContextWPtr(m_TlsContext)); + + // Push the initial data: + m_TlsContext->StoreReceivedData(a_StartTLSData.data(), a_StartTLSData.size()); + + // Start the handshake: + m_TlsContext->Handshake(); + return ""; +} + + + + + void cTCPLinkImpl::ReadCallback(bufferevent * a_BufferEvent, void * a_Self) { ASSERT(a_Self != nullptr); cTCPLinkImpl * Self = static_cast<cTCPLinkImpl *>(a_Self); + ASSERT(Self->m_BufferEvent == a_BufferEvent); ASSERT(Self->m_Callbacks != nullptr); // Read all the incoming data, in 1024-byte chunks: char data[1024]; size_t length; + auto tlsContext = Self->m_TlsContext; while ((length = bufferevent_read(a_BufferEvent, data, sizeof(data))) > 0) { - Self->m_Callbacks->OnReceivedData(data, length); + if (tlsContext != nullptr) + { + ASSERT(tlsContext->IsLink(Self)); + tlsContext->StoreReceivedData(data, length); + } + else + { + Self->ReceivedCleartextData(data, length); + } } } @@ -214,6 +322,11 @@ void cTCPLinkImpl::EventCallback(bufferevent * a_BufferEvent, short a_What, void { ASSERT(a_Self != nullptr); cTCPLinkImplPtr Self = static_cast<cTCPLinkImpl *>(a_Self)->m_Self; + if (Self == nullptr) + { + // The link has already been freed + return; + } // If an error is reported, call the error callback: if (a_What & BEV_EVENT_ERROR) @@ -262,6 +375,13 @@ void cTCPLinkImpl::EventCallback(bufferevent * a_BufferEvent, short a_What, void // If the connection has been closed, call the link callback and remove the connection: if (a_What & BEV_EVENT_EOF) { + // If running in TLS mode and there's data left in the TLS contect, report it: + auto tlsContext = Self->m_TlsContext; + if (tlsContext != nullptr) + { + tlsContext->FlushBuffers(); + } + Self->m_Callbacks->OnRemoteClosed(); if (Self->m_Server != nullptr) { @@ -357,6 +477,178 @@ void cTCPLinkImpl::DoActualShutdown(void) +bool cTCPLinkImpl::SendRaw(const void * a_Data, size_t a_Length) +{ + return (bufferevent_write(m_BufferEvent, a_Data, a_Length) == 0); +} + + + + + +void cTCPLinkImpl::ReceivedCleartextData(const char * a_Data, size_t a_Length) +{ + ASSERT(m_Callbacks != nullptr); + m_Callbacks->OnReceivedData(a_Data, a_Length); +} + + + + + +//////////////////////////////////////////////////////////////////////////////// +// cTCPLinkImpl::cLinkTlsContext: + +cTCPLinkImpl::cLinkTlsContext::cLinkTlsContext(cTCPLinkImpl & a_Link): + m_Link(a_Link) +{ +} + + + + + +void cTCPLinkImpl::cLinkTlsContext::SetSelf(cLinkTlsContextWPtr a_Self) +{ + m_Self = a_Self; +} + + + + + +void cTCPLinkImpl::cLinkTlsContext::ResetSelf(void) +{ + m_Self.reset(); +} + + + + + +void cTCPLinkImpl::cLinkTlsContext::StoreReceivedData(const char * a_Data, size_t a_NumBytes) +{ + // Hold self alive for the duration of this function + cLinkTlsContextPtr Self(m_Self); + + m_EncryptedData.append(a_Data, a_NumBytes); + + // Try to finish a pending handshake: + TryFinishHandshaking(); + + // Flush any cleartext data that can be "received": + FlushBuffers(); +} + + + + + +void cTCPLinkImpl::cLinkTlsContext::FlushBuffers(void) +{ + // Hold self alive for the duration of this function + cLinkTlsContextPtr Self(m_Self); + + // If the handshake didn't complete yet, bail out: + if (!HasHandshaken()) + { + return; + } + + char Buffer[1024]; + int NumBytes; + while ((NumBytes = ReadPlain(Buffer, sizeof(Buffer))) > 0) + { + m_Link.ReceivedCleartextData(Buffer, static_cast<size_t>(NumBytes)); + if (m_Self.expired()) + { + // The callback closed the SSL context, bail out + return; + } + } +} + + + + + +void cTCPLinkImpl::cLinkTlsContext::TryFinishHandshaking(void) +{ + // Hold self alive for the duration of this function + cLinkTlsContextPtr Self(m_Self); + + // If the handshake hasn't finished yet, retry: + if (!HasHandshaken()) + { + Handshake(); + // If the handshake succeeded, write all the queued plaintext data: + if (HasHandshaken()) + { + m_Link.GetCallbacks()->OnTlsHandshakeCompleted(); + WritePlain(m_CleartextData.data(), m_CleartextData.size()); + m_CleartextData.clear(); + } + } +} + + + + + +void cTCPLinkImpl::cLinkTlsContext::Send(const void * a_Data, size_t a_Length) +{ + // Hold self alive for the duration of this function + cLinkTlsContextPtr Self(m_Self); + + // If the handshake hasn't completed yet, queue the data: + if (!HasHandshaken()) + { + m_CleartextData.append(reinterpret_cast<const char *>(a_Data), a_Length); + TryFinishHandshaking(); + return; + } + + // The connection is all set up, write the cleartext data into the SSL context: + WritePlain(a_Data, a_Length); + FlushBuffers(); +} + + + + + +int cTCPLinkImpl::cLinkTlsContext::ReceiveEncrypted(unsigned char * a_Buffer, size_t a_NumBytes) +{ + // Hold self alive for the duration of this function + cLinkTlsContextPtr Self(m_Self); + + // If there's nothing queued in the buffer, report empty buffer: + if (m_EncryptedData.empty()) + { + return POLARSSL_ERR_NET_WANT_READ; + } + + // Copy as much data as possible to the provided buffer: + size_t BytesToCopy = std::min(a_NumBytes, m_EncryptedData.size()); + memcpy(a_Buffer, m_EncryptedData.data(), BytesToCopy); + m_EncryptedData.erase(0, BytesToCopy); + return static_cast<int>(BytesToCopy); +} + + + + + +int cTCPLinkImpl::cLinkTlsContext::SendEncrypted(const unsigned char * a_Buffer, size_t a_NumBytes) +{ + m_Link.SendRaw(a_Buffer, a_NumBytes); + return static_cast<int>(a_NumBytes); +} + + + + + //////////////////////////////////////////////////////////////////////////////// // cNetwork API: diff --git a/src/OSSupport/TCPLinkImpl.h b/src/OSSupport/TCPLinkImpl.h index bea21aeff..b54c1a2cc 100644 --- a/src/OSSupport/TCPLinkImpl.h +++ b/src/OSSupport/TCPLinkImpl.h @@ -14,6 +14,7 @@ #include "Network.h" #include <event2/event.h> #include <event2/bufferevent.h> +#include "../PolarSSL++/SslContext.h" @@ -64,9 +65,73 @@ public: virtual UInt16 GetRemotePort(void) const override { return m_RemotePort; } virtual void Shutdown(void) override; virtual void Close(void) override; + virtual AString StartTLSClient( + cX509CertPtr a_OwnCert, + cCryptoKeyPtr a_OwnPrivKey + ) override; + virtual AString StartTLSServer( + cX509CertPtr a_OwnCert, + cCryptoKeyPtr a_OwnPrivKey, + const AString & a_StartTLSData + ) override; protected: + // fwd: + class cLinkTlsContext; + typedef SharedPtr<cLinkTlsContext> cLinkTlsContextPtr; + typedef WeakPtr<cLinkTlsContext> cLinkTlsContextWPtr; + + /** Wrapper around cSslContext that is used when this link is being encrypted by SSL. */ + class cLinkTlsContext : + public cSslContext + { + cTCPLinkImpl & m_Link; + + /** Buffer for storing the incoming encrypted data until it is requested by the SSL decryptor. */ + AString m_EncryptedData; + + /** Buffer for storing the outgoing cleartext data until the link has finished handshaking. */ + AString m_CleartextData; + + /** Shared ownership of self, so that this object can keep itself alive for as long as it needs. */ + cLinkTlsContextWPtr m_Self; + + public: + cLinkTlsContext(cTCPLinkImpl & a_Link); + + /** Shares ownership of self, so that this object can keep itself alive for as long as it needs. */ + void SetSelf(cLinkTlsContextWPtr a_Self); + + /** Removes the self ownership so that we can detect the SSL closure. */ + void ResetSelf(void); + + /** Stores the specified block of data into the buffer of the data to be decrypted (incoming from remote). + Also flushes the SSL buffers by attempting to read any data through the SSL context. */ + void StoreReceivedData(const char * a_Data, size_t a_NumBytes); + + /** Tries to read any cleartext data available through the SSL, reports it in the link. */ + void FlushBuffers(void); + + /** Tries to finish handshaking the SSL. */ + void TryFinishHandshaking(void); + + /** Sends the specified cleartext data over the SSL to the remote peer. + If the handshake hasn't been completed yet, queues the data for sending when it completes. */ + void Send(const void * a_Data, size_t a_Length); + + // cSslContext overrides: + virtual int ReceiveEncrypted(unsigned char * a_Buffer, size_t a_NumBytes) override; + virtual int SendEncrypted(const unsigned char * a_Buffer, size_t a_NumBytes) override; + + /** Returns true if the context's associated TCP link is the same link as a_Link. */ + bool IsLink(cTCPLinkImpl * a_Link) + { + return (a_Link == &m_Link); + } + }; + + /** Callbacks to call when the connection is established. May be NULL if not used. Only used for outgoing connections (cNetwork::Connect()). */ cNetwork::cConnectCallbacksPtr m_ConnectCallbacks; @@ -99,6 +164,10 @@ protected: data is sent to the OS TCP stack, the socket gets shut down. */ bool m_ShouldShutdown; + /** The SSL context used for encryption, if this link uses SSL. + If valid, the link uses encryption through this context. */ + cLinkTlsContextPtr m_TlsContext; + /** Creates a new link to be queued to connect to a specified host:port. Used for outgoing connections created using cNetwork::Connect(). @@ -127,6 +196,12 @@ protected: /** Calls shutdown on the link and disables LibEvent writing. Called after all data from LibEvent buffers is sent to the OS TCP stack and shutdown() has been called before. */ void DoActualShutdown(void); + + /** Sends the data directly to the socket (without the optional TLS). */ + bool SendRaw(const void * a_Data, size_t a_Length); + + /** Called by the TLS when it has decoded a piece of incoming cleartext data from the socket. */ + void ReceivedCleartextData(const char * a_Data, size_t a_Length); }; |