From 6bea4ec988b7caeac63353fb9d2a354f2fd47dad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Mr=C3=A1zek?= Date: Sun, 8 Sep 2013 02:15:20 +0200 Subject: [PATCH] Use HttpMetaCache to minimize network use. --- CMakeLists.txt | 7 ++ MultiMC.cpp | 21 +++- MultiMC.h | 8 ++ logic/LegacyUpdate.cpp | 4 +- logic/OneSixAssets.cpp | 29 +++--- logic/OneSixUpdate.cpp | 20 ++-- logic/net/ByteArrayDownload.cpp | 62 ++++++++++++ logic/net/ByteArrayDownload.h | 24 +++++ logic/net/CacheDownload.cpp | 122 +++++++++++++++++++++++ logic/net/CacheDownload.h | 34 +++++++ logic/net/Download.h | 54 +++++++++++ logic/net/DownloadJob.cpp | 146 ++++------------------------ logic/net/DownloadJob.h | 95 +++--------------- logic/net/FileDownload.cpp | 111 +++++++++++++++++++++ logic/net/FileDownload.h | 35 +++++++ logic/net/HttpMetaCache.cpp | 167 +++++++++++++++++++++++++------- logic/net/HttpMetaCache.h | 33 +++++-- 17 files changed, 698 insertions(+), 274 deletions(-) create mode 100644 logic/net/ByteArrayDownload.cpp create mode 100644 logic/net/ByteArrayDownload.h create mode 100644 logic/net/CacheDownload.cpp create mode 100644 logic/net/CacheDownload.h create mode 100644 logic/net/Download.h create mode 100644 logic/net/FileDownload.cpp create mode 100644 logic/net/FileDownload.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 27f8eaba..cbc3d842 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -184,6 +184,10 @@ logic/ModList.h logic/InstanceLauncher.h # network stuffs +logic/net/Download.h +logic/net/FileDownload.h +logic/net/ByteArrayDownload.h +logic/net/CacheDownload.h logic/net/DownloadJob.h logic/net/HttpMetaCache.h @@ -252,6 +256,9 @@ logic/ModList.cpp logic/InstanceLauncher.cpp # network stuffs - to be moved into a depend lib ~_~ +logic/net/FileDownload.cpp +logic/net/ByteArrayDownload.cpp +logic/net/CacheDownload.cpp logic/net/DownloadJob.cpp logic/net/HttpMetaCache.cpp diff --git a/MultiMC.cpp b/MultiMC.cpp index 88eb3e62..f212d830 100644 --- a/MultiMC.cpp +++ b/MultiMC.cpp @@ -8,6 +8,7 @@ #include "logic/lists/InstanceList.h" #include "logic/lists/IconList.h" #include "logic/InstanceLauncher.h" +#include "logic/net/HttpMetaCache.h" #include "pathutils.h" @@ -112,11 +113,16 @@ MultiMC::MultiMC ( int& argc, char** argv ) // load settings initGlobalSettings(); + // and instances m_instances = new InstanceList(m_settings->get("InstanceDir").toString(),this); std::cout << "Loading Instances..." << std::endl; m_instances->loadList(); - // network manager + + // init the http meta cache + initHttpMetaCache(); + + // create the global network manager m_qnam = new QNetworkAccessManager(this); // Register meta types. @@ -137,11 +143,12 @@ MultiMC::MultiMC ( int& argc, char** argv ) MultiMC::~MultiMC() { delete m_settings; + delete m_metacache; } void MultiMC::initGlobalSettings() { - m_settings = new INISettingsObject(applicationDirPath() + "/multimc.cfg", this); + m_settings = new INISettingsObject("multimc.cfg", this); // Updates m_settings->registerSetting(new Setting("UseDevBuilds", false)); m_settings->registerSetting(new Setting("AutoUpdate", true)); @@ -189,6 +196,16 @@ void MultiMC::initGlobalSettings() m_settings->registerSetting(new Setting("TheCat", false)); } +void MultiMC::initHttpMetaCache() +{ + m_metacache = new HttpMetaCache("metacache"); + m_metacache->addBase("assets", QDir("assets").absolutePath()); + m_metacache->addBase("versions", QDir("versions").absolutePath()); + m_metacache->addBase("libraries", QDir("libraries").absolutePath()); + m_metacache->Load(); +} + + IconList* MultiMC::icons() { if ( !m_icons ) diff --git a/MultiMC.h b/MultiMC.h index 99d90b99..0c8b673b 100644 --- a/MultiMC.h +++ b/MultiMC.h @@ -4,6 +4,7 @@ #include "MultiMCVersion.h" #include "config.h" +class HttpMetaCache; class SettingsObject; class InstanceList; class IconList; @@ -55,14 +56,21 @@ public: { return m_qnam; } + + HttpMetaCache * metacache() + { + return m_metacache; + } private: void initGlobalSettings(); + void initHttpMetaCache(); private: SettingsObject * m_settings = nullptr; InstanceList * m_instances = nullptr; IconList * m_icons = nullptr; QNetworkAccessManager * m_qnam = nullptr; + HttpMetaCache * m_metacache = nullptr; Status m_status = MultiMC::Failed; MultiMCVersion m_version = {VERSION_MAJOR, VERSION_MINOR, VERSION_REVISION, VERSION_BUILD}; }; \ No newline at end of file diff --git a/logic/LegacyUpdate.cpp b/logic/LegacyUpdate.cpp index 626ad1e0..c75a0011 100644 --- a/logic/LegacyUpdate.cpp +++ b/logic/LegacyUpdate.cpp @@ -227,7 +227,9 @@ void LegacyUpdate::jarStart() QString intended_version_id = inst->intendedVersionId(); urlstr += intended_version_id + "/" + intended_version_id + ".jar"; - legacyDownloadJob.reset(new DownloadJob(QUrl(urlstr), inst->defaultBaseJar())); + auto dljob = new DownloadJob("Minecraft.jar for version " + intended_version_id); + dljob->add(QUrl(urlstr), inst->defaultBaseJar()); + legacyDownloadJob.reset(); connect(legacyDownloadJob.data(), SIGNAL(finished()), SLOT(jarFinished())); connect(legacyDownloadJob.data(), SIGNAL(failed()), SLOT(jarFailed())); connect(legacyDownloadJob.data(), SIGNAL(progress(qint64,qint64)), SLOT(updateDownloadProgress(qint64,qint64))); diff --git a/logic/OneSixAssets.cpp b/logic/OneSixAssets.cpp index c65ee607..5bdd29d7 100644 --- a/logic/OneSixAssets.cpp +++ b/logic/OneSixAssets.cpp @@ -3,6 +3,8 @@ #include #include "OneSixAssets.h" #include "net/DownloadJob.h" +#include "net/HttpMetaCache.h" +#include "MultiMC.h" inline QDomElement getDomElementByTagName(QDomElement parent, QString tagname) { @@ -65,7 +67,7 @@ void OneSixAssets::fetchXMLFinished() nuke_whitelist.clear(); auto firstJob = index_job->first(); - QByteArray ba = firstJob->m_data; + QByteArray ba = firstJob.dynamicCast()->m_data; QString xmlErrorMsg; QDomDocument doc; @@ -76,10 +78,12 @@ void OneSixAssets::fetchXMLFinished() //QRegExp etag_match(".*([a-f0-9]{32}).*"); QDomNodeList contents = doc.elementsByTagName ( "Contents" ); - DownloadJob *job = new DownloadJob(); + DownloadJob *job = new DownloadJob("Assets"); connect ( job, SIGNAL(succeeded()), SLOT(downloadFinished()) ); connect ( job, SIGNAL(failed()), SIGNAL(failed()) ); + auto metacache = MMC->metacache(); + for ( int i = 0; i < contents.length(); i++ ) { QDomElement element = contents.at ( i ).toElement(); @@ -104,22 +108,12 @@ void OneSixAssets::fetchXMLFinished() if ( sizeStr == "0" ) continue; - QString filename = fprefix + keyStr; - QFile check_file ( filename ); - QString client_etag = "nonsense"; - // if there already is a file and md5 checking is in effect and it can be opened - if ( check_file.exists() && check_file.open ( QIODevice::ReadOnly ) ) - { - // check the md5 against the expected one - client_etag = QCryptographicHash::hash ( check_file.readAll(), QCryptographicHash::Md5 ).toHex().constData(); - check_file.close(); - } - - QString trimmedEtag = etagStr.remove ( '"' ); nuke_whitelist.append ( keyStr ); - if(trimmedEtag != client_etag) + + auto entry = metacache->resolveEntry("assets", keyStr, etagStr); + if(entry->stale) { - job->add ( QUrl ( prefix + keyStr ), filename ); + job->add(QUrl(prefix + keyStr), entry); } } if(job->size()) @@ -135,7 +129,8 @@ void OneSixAssets::fetchXMLFinished() } void OneSixAssets::start() { - DownloadJob * job = new DownloadJob(QUrl ( "http://s3.amazonaws.com/Minecraft.Resources/" )); + auto job = new DownloadJob("Assets index"); + job->add(QUrl ( "http://s3.amazonaws.com/Minecraft.Resources/" )); connect ( job, SIGNAL(succeeded()), SLOT ( fetchXMLFinished() ) ); index_job.reset ( job ); job->start(); diff --git a/logic/OneSixUpdate.cpp b/logic/OneSixUpdate.cpp index 428d6ef7..ce71bde0 100644 --- a/logic/OneSixUpdate.cpp +++ b/logic/OneSixUpdate.cpp @@ -12,7 +12,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - +#include "MultiMC.h" #include "OneSixUpdate.h" #include @@ -72,7 +72,9 @@ void OneSixUpdate::versionFileStart() QString urlstr("http://s3.amazonaws.com/Minecraft.Download/versions/"); urlstr += targetVersion->descriptor + "/" + targetVersion->descriptor + ".json"; - specificVersionDownloadJob.reset(new DownloadJob(QUrl(urlstr))); + auto job = new DownloadJob("Version index"); + job->add(QUrl(urlstr)); + specificVersionDownloadJob.reset(job); connect(specificVersionDownloadJob.data(), SIGNAL(succeeded()), SLOT(versionFileFinished())); connect(specificVersionDownloadJob.data(), SIGNAL(failed()), SLOT(versionFileFailed())); connect(specificVersionDownloadJob.data(), SIGNAL(progress(qint64,qint64)), SLOT(updateDownloadProgress(qint64,qint64))); @@ -92,7 +94,7 @@ void OneSixUpdate::versionFileFinished() // FIXME: detect errors here, download to a temp file, swap QFile vfile1 (version1); vfile1.open(QIODevice::Truncate | QIODevice::WriteOnly ); - vfile1.write(DlJob->m_data); + vfile1.write(DlJob.dynamicCast()->m_data); vfile1.close(); } @@ -134,16 +136,22 @@ void OneSixUpdate::jarlibStart() QString targetstr ("versions/"); targetstr += version->id + "/" + version->id + ".jar"; - jarlibDownloadJob.reset(new DownloadJob(QUrl(urlstr), targetstr)); + auto job = new DownloadJob("Libraries for instance " + inst->name()); + job->add(QUrl(urlstr), targetstr); + jarlibDownloadJob.reset(job); auto libs = version->getActiveNativeLibs(); libs.append(version->getActiveNormalLibs()); + auto metacache = MMC->metacache(); for(auto lib: libs) { QString download_path = lib->downloadPath(); - QString storage_path = "libraries/" + lib->storagePath(); - jarlibDownloadJob->add(download_path, storage_path); + auto entry = metacache->resolveEntry("libraries", lib->storagePath()); + if(entry->stale) + { + jarlibDownloadJob->add(download_path, entry); + } } connect(jarlibDownloadJob.data(), SIGNAL(succeeded()), SLOT(jarlibFinished())); connect(jarlibDownloadJob.data(), SIGNAL(failed()), SLOT(jarlibFailed())); diff --git a/logic/net/ByteArrayDownload.cpp b/logic/net/ByteArrayDownload.cpp new file mode 100644 index 00000000..6ae3f121 --- /dev/null +++ b/logic/net/ByteArrayDownload.cpp @@ -0,0 +1,62 @@ +#include "ByteArrayDownload.h" +#include "MultiMC.h" +#include + +ByteArrayDownload::ByteArrayDownload ( QUrl url ) + : Download() +{ + m_url = url; + m_status = Job_NotStarted; +} + +void ByteArrayDownload::start() +{ + qDebug() << "Downloading " << m_url.toString(); + QNetworkRequest request ( m_url ); + auto worker = MMC->qnam(); + QNetworkReply * rep = worker->get ( request ); + + m_reply = QSharedPointer ( rep, &QObject::deleteLater ); + connect ( rep, SIGNAL ( downloadProgress ( qint64,qint64 ) ), SLOT ( downloadProgress ( qint64,qint64 ) ) ); + connect ( rep, SIGNAL ( finished() ), SLOT ( downloadFinished() ) ); + connect ( rep, SIGNAL ( error ( QNetworkReply::NetworkError ) ), SLOT ( downloadError ( QNetworkReply::NetworkError ) ) ); + connect ( rep, SIGNAL ( readyRead() ), SLOT ( downloadReadyRead() ) ); +} + +void ByteArrayDownload::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal ) +{ + emit progress (index_within_job, bytesReceived, bytesTotal ); +} + +void ByteArrayDownload::downloadError ( QNetworkReply::NetworkError error ) +{ + // error happened during download. + // TODO: log the reason why + m_status = Job_Failed; +} + +void ByteArrayDownload::downloadFinished() +{ + // if the download succeeded + if ( m_status != Job_Failed ) + { + // nothing went wrong... + m_status = Job_Finished; + m_data = m_reply->readAll(); + m_reply.clear(); + emit succeeded(index_within_job); + return; + } + // else the download failed + else + { + m_reply.clear(); + emit failed(index_within_job); + return; + } +} + +void ByteArrayDownload::downloadReadyRead() +{ + // ~_~ +} diff --git a/logic/net/ByteArrayDownload.h b/logic/net/ByteArrayDownload.h new file mode 100644 index 00000000..428b21db --- /dev/null +++ b/logic/net/ByteArrayDownload.h @@ -0,0 +1,24 @@ +#pragma once +#include "Download.h" + +class ByteArrayDownload: public Download +{ + Q_OBJECT +public: + ByteArrayDownload(QUrl url); + +public: + /// if not saving to file, downloaded data is placed here + QByteArray m_data; + +public slots: + virtual void start(); + +protected slots: + void downloadProgress(qint64 bytesReceived, qint64 bytesTotal); + void downloadError(QNetworkReply::NetworkError error); + void downloadFinished(); + void downloadReadyRead(); +}; + +typedef QSharedPointer ByteArrayDownloadPtr; diff --git a/logic/net/CacheDownload.cpp b/logic/net/CacheDownload.cpp new file mode 100644 index 00000000..c0074574 --- /dev/null +++ b/logic/net/CacheDownload.cpp @@ -0,0 +1,122 @@ +#include "MultiMC.h" +#include "CacheDownload.h" +#include + +#include +#include +#include +#include + +CacheDownload::CacheDownload (QUrl url, MetaEntryPtr entry ) + :Download(), md5sum(QCryptographicHash::Md5) +{ + m_url = url; + m_entry = entry; + m_target_path = entry->getFullPath(); + m_status = Job_NotStarted; + m_opened_for_saving = false; +} + +void CacheDownload::start() +{ + if(!m_entry->stale) + { + emit succeeded(index_within_job); + return; + } + m_output_file.setFileName(m_target_path); + // if there already is a file and md5 checking is in effect and it can be opened + if(!ensureFilePathExists(m_target_path)) + { + emit failed(index_within_job); + return; + } + qDebug() << "Downloading " << m_url.toString(); + QNetworkRequest request ( m_url ); + request.setRawHeader(QString("If-None-Match").toLatin1(), m_entry->etag.toLatin1()); + + auto worker = MMC->qnam(); + QNetworkReply * rep = worker->get ( request ); + + m_reply = QSharedPointer ( rep, &QObject::deleteLater ); + connect ( rep, SIGNAL ( downloadProgress ( qint64,qint64 ) ), SLOT ( downloadProgress ( qint64,qint64 ) ) ); + connect ( rep, SIGNAL ( finished() ), SLOT ( downloadFinished() ) ); + connect ( rep, SIGNAL ( error ( QNetworkReply::NetworkError ) ), SLOT ( downloadError ( QNetworkReply::NetworkError ) ) ); + connect ( rep, SIGNAL ( readyRead() ), SLOT ( downloadReadyRead() ) ); +} + +void CacheDownload::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal ) +{ + emit progress (index_within_job, bytesReceived, bytesTotal ); +} + +void CacheDownload::downloadError ( QNetworkReply::NetworkError error ) +{ + // error happened during download. + // TODO: log the reason why + m_status = Job_Failed; +} +void CacheDownload::downloadFinished() +{ + // if the download succeeded + if ( m_status != Job_Failed ) + { + + // nothing went wrong... + m_status = Job_Finished; + if(m_opened_for_saving) + { + // save the data to the downloadable if we aren't saving to file + m_output_file.close(); + m_entry->md5sum = md5sum.result().toHex().constData(); + } + else + { + if ( m_output_file.open ( QIODevice::ReadOnly ) ) + { + m_entry->md5sum = QCryptographicHash::hash ( m_output_file.readAll(), QCryptographicHash::Md5 ).toHex().constData(); + m_output_file.close(); + } + } + QFileInfo output_file_info(m_target_path); + + + m_entry->etag = m_reply->rawHeader("ETag").constData(); + m_entry->last_changed_timestamp = output_file_info.lastModified().toUTC().toMSecsSinceEpoch(); + m_entry->stale = false; + MMC->metacache()->updateEntry(m_entry); + + m_reply.clear(); + emit succeeded(index_within_job); + return; + } + // else the download failed + else + { + m_output_file.close(); + m_output_file.remove(); + m_reply.clear(); + emit failed(index_within_job); + return; + } +} + +void CacheDownload::downloadReadyRead() +{ + if(!m_opened_for_saving) + { + if ( !m_output_file.open ( QIODevice::WriteOnly ) ) + { + /* + * Can't open the file... the job failed + */ + m_reply->abort(); + emit failed(index_within_job); + return; + } + m_opened_for_saving = true; + } + QByteArray ba = m_reply->readAll(); + md5sum.addData(ba); + m_output_file.write ( ba ); +} diff --git a/logic/net/CacheDownload.h b/logic/net/CacheDownload.h new file mode 100644 index 00000000..f95dabd5 --- /dev/null +++ b/logic/net/CacheDownload.h @@ -0,0 +1,34 @@ +#pragma once + +#include "Download.h" +#include "HttpMetaCache.h" +#include +#include + +class CacheDownload : public Download +{ + Q_OBJECT +public: + MetaEntryPtr m_entry; + /// is the saving file already open? + bool m_opened_for_saving; + /// if saving to file, use the one specified in this string + QString m_target_path; + /// this is the output file, if any + QFile m_output_file; + /// the hash-as-you-download + QCryptographicHash md5sum; +public: + explicit CacheDownload(QUrl url, MetaEntryPtr entry); + +protected slots: + virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal); + virtual void downloadError(QNetworkReply::NetworkError error); + virtual void downloadFinished(); + virtual void downloadReadyRead(); + +public slots: + virtual void start(); +}; + +typedef QSharedPointer CacheDownloadPtr; diff --git a/logic/net/Download.h b/logic/net/Download.h new file mode 100644 index 00000000..91f09dec --- /dev/null +++ b/logic/net/Download.h @@ -0,0 +1,54 @@ +#pragma once + +#include +#include +#include +#include + + +enum JobStatus +{ + Job_NotStarted, + Job_InProgress, + Job_Finished, + Job_Failed +}; + +class Download : public QObject +{ + Q_OBJECT +protected: + explicit Download(): QObject(0){}; +public: + virtual ~Download() {}; + +public: + /// the network reply + QSharedPointer m_reply; + + /// source URL + QUrl m_url; + + /// The file's status + JobStatus m_status; + + /// index within the parent job + int index_within_job = 0; + +signals: + void started(int index); + void progress(int index, qint64 current, qint64 total); + void succeeded(int index); + void failed(int index); + +protected slots: + virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal) = 0; + virtual void downloadError(QNetworkReply::NetworkError error) = 0; + virtual void downloadFinished() = 0; + virtual void downloadReadyRead() = 0; + +public slots: + virtual void start() = 0; +}; + +typedef QSharedPointer DownloadPtr; diff --git a/logic/net/DownloadJob.cpp b/logic/net/DownloadJob.cpp index cad9ae72..cbc2c5fe 100644 --- a/logic/net/DownloadJob.cpp +++ b/logic/net/DownloadJob.cpp @@ -1,136 +1,29 @@ #include "DownloadJob.h" #include "pathutils.h" #include "MultiMC.h" +#include "FileDownload.h" +#include "ByteArrayDownload.h" +#include "CacheDownload.h" -Download::Download (QUrl url, QString target_path, QString expected_md5 ) - :Job() +ByteArrayDownloadPtr DownloadJob::add ( QUrl url ) { - m_url = url; - m_target_path = target_path; - m_expected_md5 = expected_md5; - - m_check_md5 = m_expected_md5.size(); - m_save_to_file = m_target_path.size(); - m_status = Job_NotStarted; - m_opened_for_saving = false; + ByteArrayDownloadPtr ptr (new ByteArrayDownload(url)); + ptr->index_within_job = downloads.size(); + downloads.append(ptr); + return ptr; } -void Download::start() +FileDownloadPtr DownloadJob::add ( QUrl url, QString rel_target_path) { - if ( m_save_to_file ) - { - QString filename = m_target_path; - m_output_file.setFileName ( filename ); - // if there already is a file and md5 checking is in effect and it can be opened - if ( m_output_file.exists() && m_output_file.open ( QIODevice::ReadOnly ) ) - { - // check the md5 against the expected one - QString hash = QCryptographicHash::hash ( m_output_file.readAll(), QCryptographicHash::Md5 ).toHex().constData(); - m_output_file.close(); - // skip this file if they match - if ( m_check_md5 && hash == m_expected_md5 ) - { - qDebug() << "Skipping " << m_url.toString() << ": md5 match."; - emit succeeded(index_within_job); - return; - } - else - { - m_expected_md5 = hash; - } - } - if(!ensureFilePathExists(filename)) - { - emit failed(index_within_job); - return; - } - } - qDebug() << "Downloading " << m_url.toString(); - QNetworkRequest request ( m_url ); - request.setRawHeader(QString("If-None-Match").toLatin1(), m_expected_md5.toLatin1()); - - auto worker = MMC->qnam(); - QNetworkReply * rep = worker->get ( request ); - - m_reply = QSharedPointer ( rep, &QObject::deleteLater ); - connect ( rep, SIGNAL ( downloadProgress ( qint64,qint64 ) ), SLOT ( downloadProgress ( qint64,qint64 ) ) ); - connect ( rep, SIGNAL ( finished() ), SLOT ( downloadFinished() ) ); - connect ( rep, SIGNAL ( error ( QNetworkReply::NetworkError ) ), SLOT ( downloadError ( QNetworkReply::NetworkError ) ) ); - connect ( rep, SIGNAL ( readyRead() ), SLOT ( downloadReadyRead() ) ); + FileDownloadPtr ptr (new FileDownload(url, rel_target_path)); + ptr->index_within_job = downloads.size(); + downloads.append(ptr); + return ptr; } -void Download::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal ) +CacheDownloadPtr DownloadJob::add ( QUrl url, MetaEntryPtr entry) { - emit progress (index_within_job, bytesReceived, bytesTotal ); -} - -void Download::downloadError ( QNetworkReply::NetworkError error ) -{ - // error happened during download. - // TODO: log the reason why - m_status = Job_Failed; -} - -void Download::downloadFinished() -{ - // if the download succeeded - if ( m_status != Job_Failed ) - { - // nothing went wrong... - m_status = Job_Finished; - // save the data to the downloadable if we aren't saving to file - if ( !m_save_to_file ) - { - m_data = m_reply->readAll(); - } - else - { - m_output_file.close(); - } - - //TODO: check md5 here! - m_reply.clear(); - emit succeeded(index_within_job); - return; - } - // else the download failed - else - { - if ( m_save_to_file ) - { - m_output_file.close(); - m_output_file.remove(); - } - m_reply.clear(); - emit failed(index_within_job); - return; - } -} - -void Download::downloadReadyRead() -{ - if( m_save_to_file ) - { - if(!m_opened_for_saving) - { - if ( !m_output_file.open ( QIODevice::WriteOnly ) ) - { - /* - * Can't open the file... the job failed - */ - m_reply->abort(); - emit failed(index_within_job); - return; - } - m_opened_for_saving = true; - } - m_output_file.write ( m_reply->readAll() ); - } -} - -DownloadPtr DownloadJob::add ( QUrl url, QString rel_target_path, QString expected_md5 ) -{ - DownloadPtr ptr (new Download(url, rel_target_path, expected_md5)); + CacheDownloadPtr ptr (new CacheDownload(url, entry)); ptr->index_within_job = downloads.size(); downloads.append(ptr); return ptr; @@ -139,16 +32,17 @@ DownloadPtr DownloadJob::add ( QUrl url, QString rel_target_path, QString expect void DownloadJob::partSucceeded ( int index ) { num_succeeded++; + qDebug() << m_job_name.toLocal8Bit() << " progress: " << num_succeeded << "/" << downloads.size(); if(num_failed + num_succeeded == downloads.size()) { if(num_failed) { - qDebug() << "Download JOB failed: " << this; + qDebug() << m_job_name.toLocal8Bit() << " failed."; emit failed(); } else { - qDebug() << "Download JOB succeeded: " << this; + qDebug() << m_job_name.toLocal8Bit() << " succeeded."; emit succeeded(); } } @@ -159,7 +53,7 @@ void DownloadJob::partFailed ( int index ) num_failed++; if(num_failed + num_succeeded == downloads.size()) { - qDebug() << "Download JOB failed: " << this; + qDebug() << m_job_name.toLocal8Bit() << " failed."; emit failed(); } } @@ -172,7 +66,7 @@ void DownloadJob::partProgress ( int index, qint64 bytesReceived, qint64 bytesTo void DownloadJob::start() { - qDebug() << "Download JOB started: " << this; + qDebug() << m_job_name.toLocal8Bit() << " started."; for(auto iter: downloads) { connect(iter.data(), SIGNAL(succeeded(int)), SLOT(partSucceeded(int))); diff --git a/logic/net/DownloadJob.h b/logic/net/DownloadJob.h index adeef646..71466282 100644 --- a/logic/net/DownloadJob.h +++ b/logic/net/DownloadJob.h @@ -1,98 +1,28 @@ #pragma once #include +#include "Download.h" +#include "ByteArrayDownload.h" +#include "FileDownload.h" +#include "CacheDownload.h" +#include "HttpMetaCache.h" class DownloadJob; -class Download; typedef QSharedPointer DownloadJobPtr; -typedef QSharedPointer DownloadPtr; - -enum JobStatus -{ - Job_NotStarted, - Job_InProgress, - Job_Finished, - Job_Failed -}; - -class Job : public QObject -{ - Q_OBJECT -protected: - explicit Job(): QObject(0){}; -public: - virtual ~Job() {}; - -public slots: - virtual void start() = 0; -}; - -class Download: public Job -{ - friend class DownloadJob; - Q_OBJECT -protected: - Download(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString()); -public: - /// the network reply - QSharedPointer m_reply; - /// source URL - QUrl m_url; - - /// if true, check the md5sum against a provided md5sum - /// also, if a file exists, perform an md5sum first and don't download only if they don't match - bool m_check_md5; - /// the expected md5 checksum - QString m_expected_md5; - - /// save to file? - bool m_save_to_file; - /// is the saving file already open? - bool m_opened_for_saving; - /// if saving to file, use the one specified in this string - QString m_target_path; - /// this is the output file, if any - QFile m_output_file; - /// if not saving to file, downloaded data is placed here - QByteArray m_data; - - int currentProgress = 0; - int totalProgress = 0; - - /// The file's status - JobStatus m_status; - - int index_within_job = 0; -signals: - void started(int index); - void progress(int index, qint64 current, qint64 total); - void succeeded(int index); - void failed(int index); -public slots: - virtual void start(); - -private slots: - void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);; - void downloadError(QNetworkReply::NetworkError error); - void downloadFinished(); - void downloadReadyRead(); -}; /** * A single file for the downloader/cache to process. */ -class DownloadJob : public Job +class DownloadJob : public QObject { Q_OBJECT public: - explicit DownloadJob() - :Job(){}; - explicit DownloadJob(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString()) - :Job() - { - add(url, rel_target_path, expected_md5); - }; + explicit DownloadJob(QString job_name) + :QObject(), m_job_name(job_name){}; + + ByteArrayDownloadPtr add(QUrl url); + FileDownloadPtr add(QUrl url, QString rel_target_path); + CacheDownloadPtr add(QUrl url, MetaEntryPtr entry); - DownloadPtr add(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString()); DownloadPtr operator[](int index) { return downloads[index]; @@ -119,6 +49,7 @@ private slots: void partSucceeded(int index); void partFailed(int index); private: + QString m_job_name; QList downloads; int num_succeeded = 0; int num_failed = 0; diff --git a/logic/net/FileDownload.cpp b/logic/net/FileDownload.cpp new file mode 100644 index 00000000..aad4a991 --- /dev/null +++ b/logic/net/FileDownload.cpp @@ -0,0 +1,111 @@ +#include "MultiMC.h" +#include "FileDownload.h" +#include +#include +#include + + +FileDownload::FileDownload ( QUrl url, QString target_path ) + :Download() +{ + m_url = url; + m_target_path = target_path; + m_check_md5 = false; + m_status = Job_NotStarted; + m_opened_for_saving = false; +} + +void FileDownload::start() +{ + QString filename = m_target_path; + m_output_file.setFileName ( filename ); + // if there already is a file and md5 checking is in effect and it can be opened + if ( m_output_file.exists() && m_output_file.open ( QIODevice::ReadOnly ) ) + { + // check the md5 against the expected one + QString hash = QCryptographicHash::hash ( m_output_file.readAll(), QCryptographicHash::Md5 ).toHex().constData(); + m_output_file.close(); + // skip this file if they match + if ( m_check_md5 && hash == m_expected_md5 ) + { + qDebug() << "Skipping " << m_url.toString() << ": md5 match."; + emit succeeded(index_within_job); + return; + } + else + { + m_expected_md5 = hash; + } + } + if(!ensureFilePathExists(filename)) + { + emit failed(index_within_job); + return; + } + + qDebug() << "Downloading " << m_url.toString(); + QNetworkRequest request ( m_url ); + request.setRawHeader(QString("If-None-Match").toLatin1(), m_expected_md5.toLatin1()); + + auto worker = MMC->qnam(); + QNetworkReply * rep = worker->get ( request ); + + m_reply = QSharedPointer ( rep, &QObject::deleteLater ); + connect ( rep, SIGNAL ( downloadProgress ( qint64,qint64 ) ), SLOT ( downloadProgress ( qint64,qint64 ) ) ); + connect ( rep, SIGNAL ( finished() ), SLOT ( downloadFinished() ) ); + connect ( rep, SIGNAL ( error ( QNetworkReply::NetworkError ) ), SLOT ( downloadError ( QNetworkReply::NetworkError ) ) ); + connect ( rep, SIGNAL ( readyRead() ), SLOT ( downloadReadyRead() ) ); +} + +void FileDownload::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal ) +{ + emit progress (index_within_job, bytesReceived, bytesTotal ); +} + +void FileDownload::downloadError ( QNetworkReply::NetworkError error ) +{ + // error happened during download. + // TODO: log the reason why + m_status = Job_Failed; +} + +void FileDownload::downloadFinished() +{ + // if the download succeeded + if ( m_status != Job_Failed ) + { + // nothing went wrong... + m_status = Job_Finished; + m_output_file.close(); + + m_reply.clear(); + emit succeeded(index_within_job); + return; + } + // else the download failed + else + { + m_output_file.close(); + m_reply.clear(); + emit failed(index_within_job); + return; + } +} + +void FileDownload::downloadReadyRead() +{ + if(!m_opened_for_saving) + { + if ( !m_output_file.open ( QIODevice::WriteOnly ) ) + { + /* + * Can't open the file... the job failed + */ + m_reply->abort(); + emit failed(index_within_job); + return; + } + m_opened_for_saving = true; + } + m_output_file.write ( m_reply->readAll() ); +} diff --git a/logic/net/FileDownload.h b/logic/net/FileDownload.h new file mode 100644 index 00000000..190bee58 --- /dev/null +++ b/logic/net/FileDownload.h @@ -0,0 +1,35 @@ +#pragma once + +#include "Download.h" +#include + +class FileDownload : public Download +{ + Q_OBJECT +public: + /// if true, check the md5sum against a provided md5sum + /// also, if a file exists, perform an md5sum first and don't download only if they don't match + bool m_check_md5; + /// the expected md5 checksum + QString m_expected_md5; + /// is the saving file already open? + bool m_opened_for_saving; + /// if saving to file, use the one specified in this string + QString m_target_path; + /// this is the output file, if any + QFile m_output_file; + +public: + explicit FileDownload(QUrl url, QString target_path); + +protected slots: + virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal); + virtual void downloadError(QNetworkReply::NetworkError error); + virtual void downloadFinished(); + virtual void downloadReadyRead(); + +public slots: + virtual void start(); +}; + +typedef QSharedPointer FileDownloadPtr; diff --git a/logic/net/HttpMetaCache.cpp b/logic/net/HttpMetaCache.cpp index 87741dc9..50a1136e 100644 --- a/logic/net/HttpMetaCache.cpp +++ b/logic/net/HttpMetaCache.cpp @@ -1,38 +1,136 @@ +#include "MultiMC.h" #include "HttpMetaCache.h" #include + +#include #include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +QString MetaEntry::getFullPath() +{ + return PathCombine(MMC->metacache()->getBasePath(base), path); +} + HttpMetaCache::HttpMetaCache(QString path) + :QObject() { m_index_file = path; -} -HttpMetaCache::~HttpMetaCache() -{ - Save(); + saveBatchingTimer.setSingleShot(true); + saveBatchingTimer.setTimerType(Qt::VeryCoarseTimer); + connect(&saveBatchingTimer,SIGNAL(timeout()),SLOT(SaveNow())); } -void HttpMetaCache::addEntry ( QString base, QString resource_path, QString etag ) +HttpMetaCache::~HttpMetaCache() +{ + saveBatchingTimer.stop(); + SaveNow(); +} + +void HttpMetaCache::SaveEventually() +{ + saveBatchingTimer.stop(); + saveBatchingTimer.start(30000); +} + +MetaEntryPtr HttpMetaCache::getEntry ( QString base, QString resource_path ) { // no base. no base path. can't store if(!m_entries.contains(base)) - return; - QString real_path = PathCombine(m_entries[base].base_path, resource_path); - QFileInfo finfo(real_path); - - // just ignore it, it's garbage if it's not a proper file - if(!finfo.isFile() || !finfo.isReadable()) { // TODO: log problem - return; + return MetaEntryPtr(); + } + EntryMap & map = m_entries[base]; + if(map.entry_list.contains(resource_path)) + { + return map.entry_list[resource_path]; + } + return MetaEntryPtr(); +} + +MetaEntryPtr HttpMetaCache::resolveEntry ( QString base, QString resource_path, QString expected_etag ) +{ + auto entry = getEntry(base, resource_path); + // it's not present? generate a default stale entry + if(!entry) + { + return staleEntry(base, resource_path); } - Save(); + auto & selected_base = m_entries[base]; + QString real_path = PathCombine(selected_base.base_path, resource_path); + QFileInfo finfo(real_path); + + // is the file really there? if not -> stale + if(!finfo.isFile() || !finfo.isReadable()) + { + // if the file doesn't exist, we disown the entry + selected_base.entry_list.remove(resource_path); + return staleEntry(base, resource_path); + } + + if(!expected_etag.isEmpty() && expected_etag != entry->etag) + { + // if the etag doesn't match expected, we disown the entry + selected_base.entry_list.remove(resource_path); + return staleEntry(base, resource_path); + } + + // if the file changed, check md5sum + qint64 file_last_changed = finfo.lastModified().toUTC().toMSecsSinceEpoch(); + if(file_last_changed != entry->last_changed_timestamp) + { + QFile input(real_path); + input.open(QIODevice::ReadOnly); + QString md5sum = QCryptographicHash::hash(input.readAll(), QCryptographicHash::Md5).toHex().constData(); + if(entry->md5sum != md5sum) + { + selected_base.entry_list.remove(resource_path); + return staleEntry(base, resource_path); + } + // md5sums matched... keep entry and save the new state to file + entry->last_changed_timestamp = file_last_changed; + SaveEventually(); + } + + // entry passed all the checks we cared about. + return entry; +} + +bool HttpMetaCache::updateEntry ( MetaEntryPtr stale_entry ) +{ + if(!m_entries.contains(stale_entry->base)) + { + qDebug() << "Cannot add entry with unknown base: " << stale_entry->base.toLocal8Bit(); + return false; + } + if(stale_entry->stale) + { + qDebug() << "Cannot add stale entry: " << stale_entry->getFullPath().toLocal8Bit(); + return false; + } + m_entries[stale_entry->base].entry_list[stale_entry->path] = stale_entry; + SaveEventually(); + return true; +} + +MetaEntryPtr HttpMetaCache::staleEntry(QString base, QString resource_path) +{ + auto foo = new MetaEntry; + foo->base = base; + foo->path = resource_path; + foo->stale = true; + return MetaEntryPtr(foo); } void HttpMetaCache::addBase ( QString base, QString base_root ) @@ -46,6 +144,16 @@ void HttpMetaCache::addBase ( QString base, QString base_root ) m_entries[base] = foo; } +QString HttpMetaCache::getBasePath ( QString base ) +{ + if(m_entries.contains(base)) + { + return m_entries[base].base_path; + } + return QString(); +} + + void HttpMetaCache::Load() { QFile index(m_index_file); @@ -65,12 +173,12 @@ void HttpMetaCache::Load() // read the entry array auto entries_val =root.value("entries"); - if(!version_val.isArray()) + if(!entries_val.isArray()) return; - QJsonArray array = json.array(); + QJsonArray array = entries_val.toArray(); for(auto element: array) { - if(!element.isObject()); + if(!element.isObject()) return; auto element_obj = element.toObject(); QString base = element_obj.value("base").toString(); @@ -83,11 +191,13 @@ void HttpMetaCache::Load() foo->md5sum = element_obj.value("md5sum").toString(); foo->etag = element_obj.value("etag").toString(); foo->last_changed_timestamp = element_obj.value("last_changed_timestamp").toDouble(); + // presumed innocent until closer examination + foo->stale = false; entrymap.entry_list[path] = MetaEntryPtr( foo ); } } -void HttpMetaCache::Save() +void HttpMetaCache::SaveNow() { QSaveFile tfile(m_index_file); if(!tfile.open(QIODevice::WriteOnly | QIODevice::Truncate)) @@ -118,14 +228,3 @@ void HttpMetaCache::Save() return; tfile.commit(); } - - -MetaEntryPtr HttpMetaCache::getEntryForResource ( QString base, QString resource_path ) -{ - if(!m_entries.contains(base)) - return MetaEntryPtr(); - auto & entrymap = m_entries[base]; - if(!entrymap.entry_list.contains(resource_path)) - return MetaEntryPtr(); - return entrymap.entry_list[resource_path]; -} diff --git a/logic/net/HttpMetaCache.h b/logic/net/HttpMetaCache.h index 161483ad..fac6bec3 100644 --- a/logic/net/HttpMetaCache.h +++ b/logic/net/HttpMetaCache.h @@ -2,6 +2,7 @@ #include #include #include +#include struct MetaEntry { @@ -9,23 +10,42 @@ struct MetaEntry QString path; QString md5sum; QString etag; - quint64 last_changed_timestamp = 0; + qint64 last_changed_timestamp = 0; + bool stale = true; + QString getFullPath(); }; typedef QSharedPointer MetaEntryPtr; -class HttpMetaCache +class HttpMetaCache : public QObject { + Q_OBJECT public: // supply path to the cache index file HttpMetaCache(QString path); ~HttpMetaCache(); - MetaEntryPtr getEntryForResource(QString base, QString resource_path); - void addEntry(QString base, QString resource_path, QString etag); + + // get the entry solely from the cache + // you probably don't want this, unless you have some specific caching needs. + MetaEntryPtr getEntry(QString base, QString resource_path); + + // get the entry from cache and verify that it isn't stale (within reason) + MetaEntryPtr resolveEntry(QString base, QString resource_path, QString expected_etag = QString()); + + // add a previously resolved stale entry + bool updateEntry(MetaEntryPtr stale_entry); + void addBase(QString base, QString base_root); -private: - void Save(); + + // (re)start a timer that calls SaveNow later. + void SaveEventually(); void Load(); + QString getBasePath ( QString base ); +public slots: + void SaveNow(); +private: + // create a new stale entry, given the parameters + MetaEntryPtr staleEntry(QString base, QString resource_path); struct EntryMap { QString base_path; @@ -33,4 +53,5 @@ private: }; QMap m_entries; QString m_index_file; + QTimer saveBatchingTimer; }; \ No newline at end of file