From 625fbf0f9f5318608f357aa254946df5f3201cdc Mon Sep 17 00:00:00 2001 From: Giulio Eulisse Date: Thu, 26 Sep 2019 12:20:15 +0200 Subject: [PATCH] CCDB: helper functions to handle Etag and multiple locations --- CCDB/include/CCDB/CcdbApi.h | 34 +++++++++++++++++------ CCDB/src/CcdbApi.cxx | 54 +++++++++++++++++++++++++++++++++++++ CCDB/test/testCcdbApi.cxx | 42 +++++++++++++++++++++++++++++ 3 files changed, 122 insertions(+), 8 deletions(-) diff --git a/CCDB/include/CCDB/CcdbApi.h b/CCDB/include/CCDB/CcdbApi.h index 06d764e7a40fd..db4529be2de3f 100644 --- a/CCDB/include/CCDB/CcdbApi.h +++ b/CCDB/include/CCDB/CcdbApi.h @@ -195,6 +195,32 @@ class CcdbApi //: public DatabaseInterface */ void retrieveBlob(std::string const& path, std::string const& targetdir, std::map const& metadata, long timestamp) const; + /** + * A helper function to extract an object from an existing in-memory TFile + * @param file a TFile instance + * @param objname name of serialized object + * @param cl The TClass object describing the serialized type + * @return raw pointer to created object + */ + static void* extractFromTFile(TFile& file, std::string const& objname, TClass const* cl); + + /** Get headers associated to a given CCDBEntry on the server. + * @param url the url which refers to the objects + * @param etag of the previous reply + * @param headers the headers found in the request. Will be emptied when we return false. + * @return true if the headers where updated WRT last time, false if the previous results can still be used. + */ + static bool getCCDBEntryHeaders(std::string const& url, std::string const& etag, std::vector& headers); + + /** + * Extract the possible locations for a file and check whether or not + * the current cached object is still valid. + * @param headers the headers to be parsed + * @param pfns the vector of pfns to be filled. + * @param etag the etag to be updated with the new value + */ + static void parseCCDBHeaders(std::vector const& headers, std::vector& pfns, std::string& etag); + private: /** * Initialize in local mode; Objects will be retrieved from snapshot @@ -258,14 +284,6 @@ class CcdbApi //: public DatabaseInterface */ void* extractFromLocalFile(std::string const& filename, std::string const& objname, TClass const* cl) const; - /** - * A helper function to extract an object from an existing in-memory TFile - * @param file a TFile instance - * @param objname name of serialized object - * @param cl The TClass object describing the serialized type - * @return raw pointer to created object - */ - static void* extractFromTFile(TFile& file, std::string const& objname, TClass const* cl); /** * Initialization of CURL diff --git a/CCDB/src/CcdbApi.cxx b/CCDB/src/CcdbApi.cxx index 04e684b934d94..26a7e72a69ede 100644 --- a/CCDB/src/CcdbApi.cxx +++ b/CCDB/src/CcdbApi.cxx @@ -824,6 +824,60 @@ std::vector CcdbApi::parseSubFolders(std::string const& reply) cons return folders; } +namespace +{ +size_t header_callback(char* buffer, size_t size, size_t nitems, void* userdata) +{ + std::vector* headers = static_cast*>(userdata); + auto header = std::string(buffer, size * nitems); + headers->emplace_back(std::string(header.data())); + return size * nitems; +} +} // namespace + +bool CcdbApi::getCCDBEntryHeaders(std::string const& url, std::string const& etag, std::vector& headers) +{ + auto curl = curl_easy_init(); + headers.clear(); + if (!curl) { + return true; + } + + struct curl_slist* list = nullptr; + list = curl_slist_append(list, ("If-None-Match: " + etag).c_str()); + + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list); + + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + /* get us the resource without a body! */ + curl_easy_setopt(curl, CURLOPT_NOBODY, 1L); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_callback); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, &headers); + + /* Perform the request */ + curl_easy_perform(curl); + long http_code = 0; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); + if (http_code == 304) { + return false; + } + return true; +} + +void CcdbApi::parseCCDBHeaders(std::vector const& headers, std::vector& pfns, std::string& etag) +{ + static std::string etagHeader = "ETag: "; + static std::string locationHeader = "Content-Location: "; + for (auto h : headers) { + if (h.find(etagHeader) == 0) { + etag = std::string(h.data() + etagHeader.size()); + } else if (h.find(locationHeader) == 0) { + pfns.emplace_back(std::string(h.data() + locationHeader.size(), h.size() - locationHeader.size())); + } + } +} + namespace { void traverseAndFillFolders(CcdbApi const& api, std::string const& top, std::vector& folders) diff --git a/CCDB/test/testCcdbApi.cxx b/CCDB/test/testCcdbApi.cxx index 7d409b9562981..928415de2dbc9 100644 --- a/CCDB/test/testCcdbApi.cxx +++ b/CCDB/test/testCcdbApi.cxx @@ -368,3 +368,45 @@ BOOST_AUTO_TEST_CASE(list_test, *utf::precondition(if_reachable())) countItems(s, countObjects, countSubfolders); BOOST_CHECK_EQUAL(countObjects, 1); } + +BOOST_AUTO_TEST_CASE(TestHeaderParsing) +{ + std::vector headers = { + "HTTP/1.1 200", + "Content-Location: /download/6dcb77c0-ca56-11e9-a807-200114580202", + "Date: Thu, 26 Sep 2019 08:09:20 GMT", + "Valid-Until: 1567771311999", + "Valid-From: 1567080816927", + "InitialValidityLimit: 1598616816927", + "Created: 1567080816956", + "ETag: \"6dcb77c0-ca56-11e9-a807-200114580202\"", + "Last-Modified: Thu, 29 Aug 2019 12:13:36 GMT", + "UpdatedFrom: 2001:1458:202:28:0:0:100:35", + "partName: send", + "Content-Disposition: inline;filename=\"o2::dataformats::CalibLHCphaseTOF_1567080816916.root\"", + "Accept-Ranges: bytes", + "Content-MD5: 9481c9d036660f80e21dae5943c2096f", + "Content-Type: application/octet-stream", + "Content-Length: 2097152"}; + std::vector results; + std::string etag; + CcdbApi::parseCCDBHeaders(headers, results, etag); + BOOST_CHECK_EQUAL(etag, "\"6dcb77c0-ca56-11e9-a807-200114580202\""); + BOOST_REQUIRE_EQUAL(results.size(), 1); + BOOST_CHECK_EQUAL(results[0], "/download/6dcb77c0-ca56-11e9-a807-200114580202"); +} + +BOOST_AUTO_TEST_CASE(TestFetchingHeaders, *utf::precondition(if_reachable())) +{ + std::string etag; + std::vector headers; + std::vector pfns; + auto updated = CcdbApi::getCCDBEntryHeaders("http://ccdb-test.cern.ch:8080/TOF/LHCphase/1567080816927", etag, headers); + BOOST_CHECK_EQUAL(updated, true); + BOOST_REQUIRE(headers.size() != 0); + CcdbApi::parseCCDBHeaders(headers, pfns, etag); + BOOST_REQUIRE(etag != ""); + BOOST_REQUIRE(pfns.size()); + updated = CcdbApi::getCCDBEntryHeaders("http://ccdb-test.cern.ch:8080/TOF/LHCphase/1567080816927", etag, headers); + BOOST_CHECK_EQUAL(updated, false); +}