From 3ebc7fa656dded1042bcdffffc1a6e54ec971a2a Mon Sep 17 00:00:00 2001 From: artur Date: Wed, 10 Aug 2016 15:54:50 +0300 Subject: [PATCH 1/3] asio implementation --- DataLoader.pro | 4 ++++ loader.cpp | 65 ++------------------------------------------------ loader.hpp | 24 ++++--------------- main.cpp | 1 - 4 files changed, 11 insertions(+), 83 deletions(-) diff --git a/DataLoader.pro b/DataLoader.pro index e068e3b..28e61e6 100644 --- a/DataLoader.pro +++ b/DataLoader.pro @@ -9,6 +9,10 @@ CONFIG -= app_bundle TEMPLATE = app +INCLUDEPATH += /home/artur/boost_1_61_0 + +LIBS += -L/home/artur/boost_1_61_0/stage/lib -lboost_system + SOURCES += main.cpp \ loader.cpp \ parser.cpp diff --git a/loader.cpp b/loader.cpp index fa20afb..0ce2d1b 100644 --- a/loader.cpp +++ b/loader.cpp @@ -5,71 +5,10 @@ namespace loader { - -Loader::Loader() -{ - connect(&m_network_manager, SIGNAL(error(QNetworkReply::NetworkError)), this, SLOT(OnError(QNetworkReply::NetworkError))); -} - -void Loader::load(const QUrl& folder_url) -{ - m_folder_reply = m_network_manager.get(QNetworkRequest(folder_url)); - connect(m_folder_reply, SIGNAL(finished()), this, SLOT(OnDownloadFolder())); - qDebug() << "Loading " << folder_url.path(QUrl::ComponentFormattingOption::DecodeReserved) << "\n"; -} - -void Loader::OnDownloadFolder() -{ - std::cout << "Downloaded" << std::endl; - QStringList files = m_parser.parse(*m_folder_reply); - const char* moex_ftp_url = "http://ftp.moex.com"; - for(auto elem : files) + void Loader::load(const std::string &url) { - m_urls.push_back(QUrl(moex_ftp_url + elem)); + boost::asio::ip::tcp } - connect(&m_network_manager, SIGNAL(finished(QNetworkReply*)), this, SLOT(OnDownloadFile(QNetworkReply*))); - download_files(); -} - -void Loader::download_files() -{ - if(!m_urls.isEmpty()) - { - m_curr_download = m_network_manager.get((QNetworkRequest(m_urls.front()))); - connect(m_curr_download, SIGNAL(downloadProgress(qint64, qint64)), this, SLOT(OnDownloadProgress(qint64,qint64))); - m_urls.pop_front(); - } -} - -void Loader::OnDownloadFile(QNetworkReply* reply) -{ - QString fname(reply->url().fileName()); - QFile file(fname); - file.open(QFile::ReadWrite); - file.write(reply->readAll()); - file.close(); - - download_files(); -} - -void Loader::OnDownloadProgress(qint64 bytesReceived, qint64 bytesTotal) -{ - int percent = bytesReceived * 100 / bytesTotal; - QString fname = m_curr_download->url().fileName(); - - std::cout << "\r downloading... " << fname.toStdString() << " -> " << bytesReceived << " / " << bytesTotal << " (" << percent << "%)"; -} - -void Loader::OnError(QNetworkReply::NetworkError code) -{ - std::cout << "Cause errror while downloading. Error code:" << code << std::endl; -} - -void Loader::print_status() -{ - qDebug() << m_network_manager.configuration().name() << "\n"; -} - } diff --git a/loader.hpp b/loader.hpp index af3458f..a8fb420 100644 --- a/loader.hpp +++ b/loader.hpp @@ -1,34 +1,20 @@ #ifndef LOADER_HPP #define LOADER_HPP -#include +#include #include "parser.hpp" namespace loader{ -class Loader : public QObject +class Loader { - Q_OBJECT public: - Loader(); - void print_status(); - void load(const QUrl& folder_url); -private: - QStringList get_files() const; - void download_files(); - -private slots: - void OnDownloadFolder(); - void OnDownloadFile(QNetworkReply*); - void OnDownloadProgress(qint64 bytesReceived, qint64 bytesTotal); - void OnError(QNetworkReply::NetworkError code); + void load(const std::string& url); private: - QNetworkAccessManager m_network_manager; - QNetworkReply* m_folder_reply, *m_curr_download; - parser::Parser m_parser; - QQueue m_urls; + boost::asio::io_service m_io; + }; } diff --git a/main.cpp b/main.cpp index 90d7159..7cc2c00 100644 --- a/main.cpp +++ b/main.cpp @@ -13,7 +13,6 @@ int main(int argc, char *argv[]) return 1; } loader::Loader load; - load.load(QUrl(args[1])); return a.exec(); } From ae5be5796e697928a616927caa394de1ec38b845 Mon Sep 17 00:00:00 2001 From: artur Date: Fri, 12 Aug 2016 15:35:39 +0300 Subject: [PATCH 2/3] added based asio functional --- loader.cpp | 38 +++++++++++++++++++++++++++++++++++++- loader.hpp | 16 +++++++++++++++- main.cpp | 8 +++++++- 3 files changed, 59 insertions(+), 3 deletions(-) diff --git a/loader.cpp b/loader.cpp index 0ce2d1b..60d2dfe 100644 --- a/loader.cpp +++ b/loader.cpp @@ -5,9 +5,45 @@ namespace loader { + Loader::Loader(boost::asio::io_service& io_service) : m_resolver(io_service), m_socket(io_service) + {} + void Loader::load(const std::string &url) { - boost::asio::ip::tcp + std::string::size_type pos = url.find('/'); + if(pos == std::string::npos) + { + std::cerr << "Invalid url" << std::endl; + return; + } + + std::string server_path = url.substr(pos + 2); + pos = server_path.find('/'); + if(pos == std::string::npos) + { + std::cerr << "Invalid url" << std::endl; + return; + } + + m_server = server_path.substr(0, pos - 1); + m_path = server_path.substr(pos); + + tcp::resolver::query query(m_server, "http"); + + m_resolver.async_resolve(query, boost::bind(&Loader::handle_resolve, this, boost::asio::placeholders::error, boost::asio::placeholders::iterator)); + } + + void Loader::handle_resolve(const boost::system::error_code &error, boost::asio::ip::basic_resolver::iterator endpoint_iterator) + { + if(!error) + { + // trying to establish connecting + boost::asio::async_connect(m_socket, endpoint_iterator, boost::bind(&Loader::handle_connect, this, boost::asio::placeholders::error)); + } + else + { + std::cout << "Error while resolving server name. Description: " << error.message() << std::endl; + } } } diff --git a/loader.hpp b/loader.hpp index a8fb420..be1500d 100644 --- a/loader.hpp +++ b/loader.hpp @@ -2,19 +2,33 @@ #define LOADER_HPP #include +#include + #include "parser.hpp" namespace loader{ +using boost::asio::ip::tcp; + class Loader { public: + Loader(boost::asio::io_service& io_service); void load(const std::string& url); private: - boost::asio::io_service m_io; + void handle_resolve(const boost::system::error_code& error, tcp::resolver::iterator endpoint_iterator); + void handle_connect(const boost::system::error_code& error); + void handle_write_request(const boost::system::error_code& error); + void handle_read_status(const boost::system::error_code& error); + void handle_read_header(const boost::system::error_code& error); + void handle_read_content(const boost::system::error_code& error); +private: + boost::asio::ip::tcp::resolver m_resolver; + boost::asio::ip::tcp::socket m_socket; + std::string m_server, m_path; }; } diff --git a/main.cpp b/main.cpp index 7cc2c00..87b2416 100644 --- a/main.cpp +++ b/main.cpp @@ -12,7 +12,13 @@ int main(int argc, char *argv[]) std::cerr << "usage: DataLoader " << std::endl; return 1; } - loader::Loader load; + boost::asio::io_service io_service; + + loader::Loader load(io_service); + + load.load(argv[1]); + + io_service.run(); return a.exec(); } From 1da2594307eb5cbbf1a40ffde5ed12bda319fa25 Mon Sep 17 00:00:00 2001 From: artur Date: Tue, 16 Aug 2016 01:40:42 +0300 Subject: [PATCH 3/3] implemented connect, read status, headers, content --- loader.cpp | 113 ++++++++++++++++++++++++++++++++++++++++++++++++++++- loader.hpp | 2 + 2 files changed, 114 insertions(+), 1 deletion(-) diff --git a/loader.cpp b/loader.cpp index 60d2dfe..f790bbd 100644 --- a/loader.cpp +++ b/loader.cpp @@ -25,7 +25,7 @@ namespace loader return; } - m_server = server_path.substr(0, pos - 1); + m_server = server_path.substr(0, pos); m_path = server_path.substr(pos); tcp::resolver::query query(m_server, "http"); @@ -45,6 +45,117 @@ namespace loader std::cout << "Error while resolving server name. Description: " << error.message() << std::endl; } } + + void Loader::handle_connect(const boost::system::error_code& err) + { + if(!err) + { + std::cout << "connected to " << m_server << std::endl; + + std::ostream request_stream(&m_request); + request_stream << "GET " << m_path << " HTTP/1.1\r\n"; + request_stream << "Host: " << m_server << "\r\n"; + request_stream << "Accept: */*\r\n"; + request_stream << "Connect: keep-alive\r\n"; + + std::cout << "request: " << &m_request << std::endl; + + boost::asio::async_write(m_socket, m_request, boost::bind(&Loader::handle_write_request, this, boost::asio::placeholders::error)); + } + else + { + std::cout << "Couldn`t connect to" << m_server << " . Error: " << err.message() << std::endl; + } + } + + void Loader::handle_write_request(const boost::system::error_code& err) + { + if(!err) + { + boost::asio::async_read_until(m_socket, m_reply, "\r\n", boost::bind(&Loader::handle_read_status, this, boost::asio::placeholders::error)); + } + else + { + std::cout << "Cant send request. Message: " << err.message() << std::endl; + } + } + + void Loader::handle_read_status(const boost::system::error_code& err) + { + if(!err) + { + std::istream reply_stream(&m_reply); + + std::string http_version; + reply_stream >> http_version; + unsigned short status; + reply_stream >> status; + std::string status_message; + std::getline(reply_stream, status_message); + if(!reply_stream || http_version.substr(0, 5) != "HTTP/") + { + std::cout << "invalid response" << std::endl; + return; + } + if(status != 200) + { + std::cout << "Response returned witch sttus code " << status << std::endl; + return; + } + + boost::asio::async_read_until(m_socket, m_reply, "\r\n\r\n", boost::bind(&Loader::handle_read_header, this, boost::asio::placeholders::error)); + } + else + { + std::cout << err.message() << std::endl; + } + } + + void Loader::handle_read_header(const boost::system::error_code& err) + { + if(!err) + { + // Process the response headers. + std::istream response_stream(&m_reply); + std::string header; + while (std::getline(response_stream, header) && header != "\r") + std::cout << header << "\n"; + std::cout << "\n"; + + // Write whatever content we already have to output. + if (m_reply.size() > 0) + std::cout << &m_reply; + + // Start reading remaining data until EOF. + boost::asio::async_read(m_socket, m_reply, + boost::asio::transfer_at_least(1), + boost::bind(&Loader::handle_read_content, this, + boost::asio::placeholders::error)); + } + else + { + std::cout << err.message() << std::endl; + } + } + + void Loader::handle_read_content(const boost::system::error_code& err) + { + if(!err) + { + // Write all of the data that has been read so far. + std::cout << &m_reply; + + // Continue reading remaining data until EOF. + boost::asio::async_read(m_socket, m_reply, + boost::asio::transfer_at_least(1), + boost::bind(&Loader::handle_read_content, this, + boost::asio::placeholders::error)); + } + else if(err != boost::asio::error::eof) + { + std::cout << "Error: " << err.message() << std::endl; + } + } } diff --git a/loader.hpp b/loader.hpp index be1500d..10db8f9 100644 --- a/loader.hpp +++ b/loader.hpp @@ -29,6 +29,8 @@ class Loader boost::asio::ip::tcp::resolver m_resolver; boost::asio::ip::tcp::socket m_socket; std::string m_server, m_path; + boost::asio::streambuf m_request; + boost::asio::streambuf m_reply; }; }