Use asset sizes from the index to make the progress bar better

CMakeLists.txt

@@ -300,8 +300,6 @@ logic/net/NetJob.h
 logic/net/NetJob.cpp
 logic/net/HttpMetaCache.h
 logic/net/HttpMetaCache.cpp
-logic/net/S3ListBucket.h
-logic/net/S3ListBucket.cpp
 logic/net/PasteUpload.h
 logic/net/PasteUpload.cpp
 logic/net/URLConstants.h

logic/OneSixUpdate.cpp

@@ -243,6 +243,7 @@ void OneSixUpdate::assetIndexFinished()
 			auto objectDL = MD5EtagDownload::make(
 				QUrl("http://" + URLConstants::RESOURCE_BASE + objectName),
 				objectFile.filePath());
+			objectDL->m_total_progress = object.size;
 			dls.append(objectDL);
 		}
 	}

logic/net/ByteArrayDownload.cpp

@@ -42,7 +42,9 @@ void ByteArrayDownload::start()
 
 void ByteArrayDownload::downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
 {
-	emit progress(index_within_job, bytesReceived, bytesTotal);
+	m_total_progress = bytesTotal;
+	m_progress = bytesReceived;
+	emit progress(m_index_within_job, bytesReceived, bytesTotal);
 }
 
 void ByteArrayDownload::downloadError(QNetworkReply::NetworkError error)
@@ -62,14 +64,14 @@ void ByteArrayDownload::downloadFinished()
 		m_status = Job_Finished;
 		m_data = m_reply->readAll();
 		m_reply.reset();
-		emit succeeded(index_within_job);
+		emit succeeded(m_index_within_job);
 		return;
 	}
 	// else the download failed
 	else
 	{
 		m_reply.reset();
-		emit failed(index_within_job);
+		emit failed(m_index_within_job);
 		return;
 	}
 }

logic/net/CacheDownload.cpp

@@ -35,14 +35,14 @@ void CacheDownload::start()
 {
 	if (!m_entry->stale)
 	{
-		emit succeeded(index_within_job);
+		emit succeeded(m_index_within_job);
 		return;
 	}
 	m_output_file.setFileName(m_target_path);
 	// if there already is a file and md5 checking is in effect and it can be opened
 	if (!ensureFilePathExists(m_target_path))
 	{
-		emit failed(index_within_job);
+		emit failed(m_index_within_job);
 		return;
 	}
 	QLOG_INFO() << "Downloading " << m_url.toString();
@@ -69,7 +69,9 @@ void CacheDownload::start()
 
 void CacheDownload::downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
 {
-	emit progress(index_within_job, bytesReceived, bytesTotal);
+	m_total_progress = bytesTotal;
+	m_progress = bytesReceived;
+	emit progress(m_index_within_job, bytesReceived, bytesTotal);
 }
 
 void CacheDownload::downloadError(QNetworkReply::NetworkError error)
@@ -116,7 +118,7 @@ void CacheDownload::downloadFinished()
 		MMC->metacache()->updateEntry(m_entry);
 
 		m_reply.reset();
-		emit succeeded(index_within_job);
+		emit succeeded(m_index_within_job);
 		return;
 	}
 	// else the download failed
@@ -125,7 +127,7 @@ void CacheDownload::downloadFinished()
 		m_output_file.close();
 		m_output_file.remove();
 		m_reply.reset();
-		emit failed(index_within_job);
+		emit failed(m_index_within_job);
 		return;
 	}
 }
@@ -140,7 +142,7 @@ void CacheDownload::downloadReadyRead()
 			* Can't open the file... the job failed
 			*/
 			m_reply->abort();
-			emit failed(index_within_job);
+			emit failed(m_index_within_job);
 			return;
 		}
 	}

logic/net/ForgeMirrors.cpp

@@ -68,7 +68,7 @@ void ForgeMirrors::deferToFixedList()
 					  "https://www.creeperhost.net/link.php?id=1",
 					  "http://new.creeperrepo.net/forge/maven/"});
 	injectDownloads();
-	emit succeeded(index_within_job);
+	emit succeeded(m_index_within_job);
 }
 
 void ForgeMirrors::parseMirrorList()
@@ -88,7 +88,7 @@ void ForgeMirrors::parseMirrorList()
 	if(!m_mirrors.size())
 		deferToFixedList();
 	injectDownloads();
-	emit succeeded(index_within_job);
+	emit succeeded(m_index_within_job);
 }
 
 void ForgeMirrors::injectDownloads()
@@ -108,7 +108,9 @@ void ForgeMirrors::injectDownloads()
 
 void ForgeMirrors::downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
 {
-	emit progress(index_within_job, bytesReceived, bytesTotal);
+	m_total_progress = bytesTotal;
+	m_progress = bytesReceived;
+	emit progress(m_index_within_job, bytesReceived, bytesTotal);
 }
 
 void ForgeMirrors::downloadReadyRead()

logic/net/ForgeXzDownload.cpp

@@ -44,20 +44,20 @@ void ForgeXzDownload::start()
 	if (!m_entry->stale)
 	{
 		m_status = Job_Finished;
-		emit succeeded(index_within_job);
+		emit succeeded(m_index_within_job);
 		return;
 	}
 	// can we actually create the real, final file?
 	if (!ensureFilePathExists(m_target_path))
 	{
 		m_status = Job_Failed;
-		emit failed(index_within_job);
+		emit failed(m_index_within_job);
 		return;
 	}
 	if (m_mirrors.empty())
 	{
 		m_status = Job_Failed;
-		emit failed(index_within_job);
+		emit failed(m_index_within_job);
 		return;
 	}
 
@@ -80,7 +80,9 @@ void ForgeXzDownload::start()
 
 void ForgeXzDownload::downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
 {
-	emit progress(index_within_job, bytesReceived, bytesTotal);
+	m_total_progress = bytesTotal;
+	m_progress = bytesReceived;
+	emit progress(m_index_within_job, bytesReceived, bytesTotal);
 }
 
 void ForgeXzDownload::downloadError(QNetworkReply::NetworkError error)
@@ -100,7 +102,7 @@ void ForgeXzDownload::failAndTryNextMirror()
 		m_mirror_index = next;
 
 	updateUrl();
-	emit failed(index_within_job);
+	emit failed(m_index_within_job);
 }
 
 void ForgeXzDownload::updateUrl()
@@ -148,7 +150,7 @@ void ForgeXzDownload::downloadFinished()
 			m_status = Job_Failed;
 			m_pack200_xz_file.remove();
 			m_reply.reset();
-			emit failed(index_within_job);
+			emit failed(m_index_within_job);
 			return;
 		}
 	}
@@ -175,7 +177,7 @@ void ForgeXzDownload::downloadReadyRead()
 			* Can't open the file... the job failed
 			*/
 			m_reply->abort();
-			emit failed(index_within_job);
+			emit failed(m_index_within_job);
 			return;
 		}
 	}
@@ -347,5 +349,5 @@ void ForgeXzDownload::decompressAndInstall()
 	MMC->metacache()->updateEntry(m_entry);
 
 	m_reply.reset();
-	emit succeeded(index_within_job);
+	emit succeeded(m_index_within_job);
 }

logic/net/MD5EtagDownload.cpp

@@ -44,7 +44,7 @@ void MD5EtagDownload::start()
 		if (m_check_md5 && hash == m_expected_md5)
 		{
 			QLOG_INFO() << "Skipping " << m_url.toString() << ": md5 match.";
-			emit succeeded(index_within_job);
+			emit succeeded(m_index_within_job);
 			return;
 		}
 		else
@@ -54,7 +54,7 @@ void MD5EtagDownload::start()
 	}
 	if (!ensureFilePathExists(filename))
 	{
-		emit failed(index_within_job);
+		emit failed(m_index_within_job);
 		return;
 	}
 
@@ -68,7 +68,7 @@ void MD5EtagDownload::start()
 	// Plus, this way, we don't end up starting a download for a file we can't open.
 	if (!m_output_file.open(QIODevice::WriteOnly))
 	{
-		emit failed(index_within_job);
+		emit failed(m_index_within_job);
 		return;
 	}
 
@@ -86,7 +86,9 @@ void MD5EtagDownload::start()
 
 void MD5EtagDownload::downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
 {
-	emit progress(index_within_job, bytesReceived, bytesTotal);
+	m_total_progress = bytesTotal;
+	m_progress = bytesReceived;
+	emit progress(m_index_within_job, bytesReceived, bytesTotal);
 }
 
 void MD5EtagDownload::downloadError(QNetworkReply::NetworkError error)
@@ -107,7 +109,7 @@ void MD5EtagDownload::downloadFinished()
 
 		QLOG_INFO() << "Finished " << m_url.toString() << " got " << m_reply->rawHeader("ETag").constData();
 		m_reply.reset();
-		emit succeeded(index_within_job);
+		emit succeeded(m_index_within_job);
 		return;
 	}
 	// else the download failed
@@ -115,7 +117,7 @@ void MD5EtagDownload::downloadFinished()
 	{
 		m_output_file.close();
 		m_reply.reset();
-		emit failed(index_within_job);
+		emit failed(m_index_within_job);
 		return;
 	}
 }
@@ -130,7 +132,7 @@ void MD5EtagDownload::downloadReadyRead()
 			* Can't open the file... the job failed
 			*/
 			m_reply->abort();
-			emit failed(index_within_job);
+			emit failed(m_index_within_job);
 			return;
 		}
 	}

logic/net/NetAction.h

@@ -38,6 +38,19 @@ protected:
 public:
 	virtual ~NetAction() {};
 
+public:
+	virtual qint64 totalProgress() const
+	{
+		return m_total_progress;
+	}
+	virtual qint64 currentProgress() const
+	{
+		return m_progress;
+	}
+	virtual qint64 numberOfFailures() const
+	{
+		return m_failures;
+	}
 public:
 	/// the network reply
 	std::shared_ptr<QNetworkReply> m_reply;
@@ -46,10 +59,16 @@ public:
 	QUrl m_url;
 
 	/// The file's status
-	JobStatus m_status;
+	JobStatus m_status = Job_NotStarted;
 
 	/// index within the parent job
-	int index_within_job = 0;
+	int m_index_within_job = 0;
+
+	qint64 m_progress = 0;
+	qint64 m_total_progress = 1;
+
+	/// number of failures up to this point
+	int m_failures = 0;
 
 signals:
 	void started(int index);

logic/net/NetJob.h

@@ -36,10 +36,16 @@ public:
 	template <typename T> bool addNetAction(T action)
 	{
 		NetActionPtr base = std::static_pointer_cast<NetAction>(action);
-		base->index_within_job = downloads.size();
+		base->m_index_within_job = downloads.size();
 		downloads.append(action);
-		parts_progress.append(part_info());
+		part_info pi;
-		total_progress++;
+		{
+			pi.current_progress = base->currentProgress();
+			pi.total_progress = base->totalProgress();
+			pi.failures = base->numberOfFailures();
+		}
+		parts_progress.append(pi);
+		total_progress += pi.total_progress;
 		// if this is already running, the action needs to be started right away!
 		if (isRunning())
 		{

logic/net/PasteUpload.cpp

@@ -78,7 +78,9 @@ bool PasteUpload::parseResult(QJsonDocument doc, QString *parseError)
 		parseError = new QString(object.value("error").toString());
 		return false;
 	}
+	// FIXME: not the place for GUI things.
 	QString pasteUrl = object.value("paste").toObject().value("link").toString();
 	QDesktopServices::openUrl(pasteUrl);
 	return true;
 }
+

logic/net/S3ListBucket.cpp

@@ -1,175 +0,0 @@
-/* Copyright 2013 MultiMC Contributors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "S3ListBucket.h"
-#include "MultiMC.h"
-#include "logger/QsLog.h"
-#include <QUrlQuery>
-#include <qxmlstream.h>
-#include <QDomDocument>
-
-inline QDomElement getDomElementByTagName(QDomElement parent, QString tagname)
-{
-	QDomNodeList elementList = parent.elementsByTagName(tagname);
-	if (elementList.count())
-		return elementList.at(0).toElement();
-	else
-		return QDomElement();
-}
-
-S3ListBucket::S3ListBucket(QUrl url) : NetAction()
-{
-	m_url = url;
-	m_status = Job_NotStarted;
-}
-
-void S3ListBucket::start()
-{
-	QUrl finalUrl = m_url;
-	if (current_marker.size())
-	{
-		QUrlQuery query;
-		query.addQueryItem("marker", current_marker);
-		finalUrl.setQuery(query);
-	}
-	QNetworkRequest request(finalUrl);
-	request.setHeader(QNetworkRequest::UserAgentHeader, "MultiMC/5.0 (Uncached)");
-	auto worker = MMC->qnam();
-	QNetworkReply *rep = worker->get(request);
-
-	m_reply = std::shared_ptr<QNetworkReply>(rep);
-	connect(rep, SIGNAL(downloadProgress(qint64, qint64)),
-			SLOT(downloadProgress(qint64, qint64)));
-	connect(rep, SIGNAL(finished()), SLOT(downloadFinished()));
-	connect(rep, SIGNAL(error(QNetworkReply::NetworkError)),
-			SLOT(downloadError(QNetworkReply::NetworkError)));
-	connect(rep, SIGNAL(readyRead()), SLOT(downloadReadyRead()));
-}
-
-void S3ListBucket::downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
-{
-	emit progress(index_within_job, bytesSoFar + bytesReceived, bytesSoFar + bytesTotal);
-}
-
-void S3ListBucket::downloadError(QNetworkReply::NetworkError error)
-{
-	// error happened during download.
-	QLOG_ERROR() << "Error getting URL:" << m_url.toString().toLocal8Bit()
-				 << "Network error: " << error;
-	m_status = Job_Failed;
-}
-
-void S3ListBucket::processValidReply()
-{
-	QLOG_TRACE() << "GOT: " << m_url.toString() << " marker:" << current_marker;
-	auto readContents = [&](QXmlStreamReader & xml)
-	{
-		QString Key, ETag, Size;
-		while (!(xml.tokenType() == QXmlStreamReader::EndElement && xml.name() == "Contents"))
-		{
-			if (xml.tokenType() == QXmlStreamReader::StartElement)
-			{
-				if (xml.name() == "Key")
-				{
-					Key = xml.readElementText();
-				}
-				if (xml.name() == "ETag")
-				{
-					ETag = xml.readElementText();
-				}
-				if (xml.name() == "Size")
-				{
-					Size = xml.readElementText();
-				}
-			}
-			xml.readNext();
-		}
-		if (xml.error() != QXmlStreamReader::NoError)
-			return;
-		objects.append({Key, ETag, Size.toLongLong()});
-	};
-
-	// nothing went wrong...
-	QByteArray ba = m_reply->readAll();
-
-	QString xmlErrorMsg;
-
-	bool is_truncated = false;
-	QXmlStreamReader xml(ba);
-	while (!xml.atEnd() && !xml.hasError())
-	{
-		/* Read next element.*/
-		QXmlStreamReader::TokenType token = xml.readNext();
-		/* If token is just StartDocument, we'll go to next.*/
-		if (token == QXmlStreamReader::StartDocument)
-		{
-			continue;
-		}
-		if (token == QXmlStreamReader::StartElement)
-		{
-			/* If it's named person, we'll dig the information from there.*/
-			if (xml.name() == "Contents")
-			{
-				readContents(xml);
-			}
-			else if (xml.name() == "IsTruncated")
-			{
-				is_truncated = (xml.readElementText() == "true");
-			}
-		}
-	}
-	if (xml.hasError())
-	{
-		QLOG_ERROR() << "Failed to process s3.amazonaws.com/Minecraft.Resources. XML error:"
-					 << xml.errorString() << ba;
-		emit failed(index_within_job);
-		return;
-	}
-	if (is_truncated)
-	{
-		current_marker = objects.last().Key;
-		bytesSoFar += m_reply->size();
-		m_reply.reset();
-		start();
-	}
-	else
-	{
-		m_status = Job_Finished;
-		m_reply.reset();
-		emit succeeded(index_within_job);
-	}
-	return;
-}
-
-void S3ListBucket::downloadFinished()
-{
-	// if the download succeeded
-	if (m_status != Job_Failed)
-	{
-		processValidReply();
-	}
-	// else the download failed
-	else
-	{
-		m_reply.reset();
-		emit failed(index_within_job);
-		return;
-	}
-}
-
-void S3ListBucket::downloadReadyRead()
-{
-	// ~_~
-}

logic/net/S3ListBucket.h

@@ -1,57 +0,0 @@
-/* Copyright 2013 MultiMC Contributors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-#include "NetAction.h"
-
-struct S3Object
-{
-	QString Key;
-	QString ETag;
-	qlonglong size;
-};
-
-typedef std::shared_ptr<class S3ListBucket> S3ListBucketPtr;
-class S3ListBucket : public NetAction
-{
-	Q_OBJECT
-public:
-	S3ListBucket(QUrl url);
-	static S3ListBucketPtr make(QUrl url)
-	{
-		return S3ListBucketPtr(new S3ListBucket(url));
-	}
-
-public:
-	QList<S3Object> objects;
-
-public
-slots:
-	virtual void start() override;
-
-protected
-slots:
-	virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal) override;
-	virtual void downloadError(QNetworkReply::NetworkError error) override;
-	virtual void downloadFinished() override;
-	virtual void downloadReadyRead() override;
-
-private:
-	void processValidReply();
-
-private:
-	qint64 bytesSoFar = 0;
-	QString current_marker;
-};