feat: improve metadata gen. networking and performance

This makes the metadata generation code a lot messier and harder to use,
but there's not really much else that can be done about it while
preserving all it's capabilities :(

At least we now have speed

Signed-off-by: flow <flowlnlnln@gmail.com>

launcher/modplatform/CheckUpdateTask.h

@@ -13,7 +13,7 @@ class CheckUpdateTask : public Task {
 
    public:
     CheckUpdateTask(std::list<Mod>& mods, std::list<Version>& mcVersions, ModAPI::ModLoaderTypes loaders, std::shared_ptr<ModFolderModel> mods_folder)
-        : m_mods(mods), m_game_versions(mcVersions), m_loaders(loaders), m_mods_folder(mods_folder) {};
+        : Task(nullptr), m_mods(mods), m_game_versions(mcVersions), m_loaders(loaders), m_mods_folder(mods_folder) {};
 
     struct UpdatableMod {
         QString name;

launcher/modplatform/EnsureMetadataTask.cpp

@@ -19,226 +19,502 @@ static ModPlatform::ProviderCapabilities ProviderCaps;
 static ModrinthAPI modrinth_api;
 static FlameAPI flame_api;
 
-EnsureMetadataTask::EnsureMetadataTask(Mod& mod, QDir& dir, bool try_all, ModPlatform::Provider prov)
+EnsureMetadataTask::EnsureMetadataTask(Mod& mod, QDir dir, ModPlatform::Provider prov) : Task(nullptr), m_index_dir(dir), m_provider(prov)
-    : m_mod(mod), m_index_dir(dir), m_provider(prov), m_try_all(try_all)
+{
-{}
+    auto hash = getHash(mod);
+    if (hash.isEmpty())
+        emitFail(mod);
+    else
+        m_mods.insert(hash, mod);
+}
+
+EnsureMetadataTask::EnsureMetadataTask(std::list<Mod>& mods, QDir dir, ModPlatform::Provider prov)
+    : Task(nullptr), m_index_dir(dir), m_provider(prov)
+{
+    for (auto& mod : mods) {
+        if (!mod.valid()) {
+            emitFail(mod);
+            continue;
+        }
+
+        auto hash = getHash(mod);
+        if (hash.isEmpty()) {
+            emitFail(mod);
+            continue;
+        }
+
+        m_mods.insert(hash, mod);
+    }
+}
+
+QString EnsureMetadataTask::getHash(Mod& mod)
+{
+    /* Here we create a mapping hash -> mod, because we need that relationship to parse the API routes */
+    QByteArray jar_data;
+    try {
+        jar_data = FS::read(mod.fileinfo().absoluteFilePath());
+    } catch (FS::FileSystemException& e) {
+        qCritical() << QString("Failed to open / read JAR file of %1").arg(mod.name());
+        qCritical() << QString("Reason: ") << e.cause();
+
+        return {};
+    }
+
+    switch (m_provider) {
+        case ModPlatform::Provider::MODRINTH: {
+            auto hash_type = ProviderCaps.hashType(ModPlatform::Provider::MODRINTH).first();
+
+            return QString(ProviderCaps.hash(ModPlatform::Provider::MODRINTH, jar_data, hash_type).toHex());
+        }
+        case ModPlatform::Provider::FLAME: {
+            QByteArray jar_data_treated;
+            for (char c : jar_data) {
+                // CF-specific
+                if (!(c == 9 || c == 10 || c == 13 || c == 32))
+                    jar_data_treated.push_back(c);
+            }
+
+            return QString::number(MurmurHash2(jar_data_treated, jar_data_treated.length()));
+        }
+    }
+
+    return {};
+}
 
 bool EnsureMetadataTask::abort()
 {
-    return m_task_handler->abort();
+    // Prevent sending signals to a dead object
+    disconnect(this, 0, 0, 0);
+
+    if (m_current_task)
+        return m_current_task->abort();
+    return true;
 }
 
 void EnsureMetadataTask::executeTask()
 {
-    // They already have the right metadata :o
+    setStatus(tr("Checking if mods have metadata..."));
-    if (m_mod.status() != ModStatus::NoMetadata && m_mod.metadata() && m_mod.metadata()->provider == m_provider) {
+
-        emitReady();
+    for (auto mod : m_mods) {
-        return;
+        if (!mod.valid())
+            continue;
+
+        // They already have the right metadata :o
+        if (mod.status() != ModStatus::NoMetadata && mod.metadata() && mod.metadata()->provider == m_provider) {
+            qDebug() << "Mod" << mod.name() << "already has metadata!";
+            emitReady(mod);
+            return;
+        }
+
+        // Folders don't have metadata
+        if (mod.type() == Mod::MOD_FOLDER) {
+            emitReady(mod);
+            return;
+        }
     }
 
-    // Folders don't have metadata
+    NetJob::Ptr version_task;
-    if (m_mod.type() == Mod::MOD_FOLDER) {
-        emitReady();
-        return;
-    }
-
-    setStatus(tr("Generating %1's metadata...").arg(m_mod.name()));
-    qDebug() << QString("Generating %1's metadata...").arg(m_mod.name());
-
-    QByteArray jar_data;
-
-    try {
-        jar_data = FS::read(m_mod.fileinfo().absoluteFilePath());
-    } catch (FS::FileSystemException& e) {
-        qCritical() << QString("Failed to open / read JAR file of %1").arg(m_mod.name());
-        qCritical() << QString("Reason: ") << e.cause();
-
-        emitFail();
-        return;
-    }
-
-    auto tsk = new MultipleOptionsTask(nullptr, "GetMetadataTask");
 
     switch (m_provider) {
         case (ModPlatform::Provider::MODRINTH):
-            modrinthEnsureMetadata(*tsk, jar_data);
+            version_task = modrinthVersionsTask();
-            if (m_try_all)
-                flameEnsureMetadata(*tsk, jar_data);
-
             break;
         case (ModPlatform::Provider::FLAME):
-            flameEnsureMetadata(*tsk, jar_data);
+            version_task = flameVersionsTask();
-            if (m_try_all)
-                modrinthEnsureMetadata(*tsk, jar_data);
-
             break;
     }
 
-    connect(tsk, &MultipleOptionsTask::finished, this, [tsk] { tsk->deleteLater(); });
+    auto invalidade_leftover = [this] {
-    connect(tsk, &MultipleOptionsTask::failed, [this] {
+        QMutableHashIterator<QString, Mod> mods_iter(m_mods);
-        qCritical() << QString("Download of %1's metadata failed").arg(m_mod.name());
+        while (mods_iter.hasNext()) {
+            auto mod = mods_iter.next();
+            emitFail(mod.value());
+        }
 
-        emitFail();
+        emitSucceeded();
-    });
+    };
-    connect(tsk, &MultipleOptionsTask::succeeded, this, &EnsureMetadataTask::emitReady);
 
-    m_task_handler = tsk;
+    connect(version_task.get(), &Task::finished, this, [this, invalidade_leftover] {
+        NetJob::Ptr project_task;
 
-    tsk->start();
+        switch (m_provider) {
-}
+            case (ModPlatform::Provider::MODRINTH):
+                project_task = modrinthProjectsTask();
+                break;
+            case (ModPlatform::Provider::FLAME):
+                project_task = flameProjectsTask();
+                break;
+        }
 
-void EnsureMetadataTask::emitReady()
+        if (!project_task) {
-{
+            invalidade_leftover();
-    emit metadataReady();
-    emitSucceeded();
-}
-
-void EnsureMetadataTask::emitFail()
-{
-    qDebug() << QString("Failed to generate metadata for %1").arg(m_mod.name());
-    emit metadataFailed();
-    //emitFailed(tr("Failed to generate metadata for %1").arg(m_mod.name()));
-    emitSucceeded();
-}
-
-void EnsureMetadataTask::modrinthEnsureMetadata(SequentialTask& tsk, QByteArray& jar_data)
-{
-    // Modrinth currently garantees that some hash types will always be present.
-    // But let's be sure and cover all cases anyways :)
-    for (auto hash_type : ProviderCaps.hashType(ModPlatform::Provider::MODRINTH)) {
-        auto* response = new QByteArray();
-        auto hash = QString(ProviderCaps.hash(ModPlatform::Provider::MODRINTH, jar_data, hash_type).toHex());
-        auto ver_task = modrinth_api.currentVersion(hash, hash_type, response);
-
-        // Prevents unfortunate timings when aborting the task
-        if (!ver_task)
             return;
+        }
 
-        connect(ver_task.get(), &NetJob::succeeded, this, [this, ver_task, response] {
+        connect(project_task.get(), &Task::finished, this, [=] {
-            QJsonParseError parse_error{};
+            invalidade_leftover();
-            QJsonDocument doc = QJsonDocument::fromJson(*response, &parse_error);
+            project_task->deleteLater();
-            if (parse_error.error != QJsonParseError::NoError) {
+            m_current_task = nullptr;
-                qWarning() << "Error while parsing JSON response from " << m_mod.name() << " at " << parse_error.offset
-                           << " reason: " << parse_error.errorString();
-                qWarning() << *response;
-
-                ver_task->failed(parse_error.errorString());
-                return;
-            }
-
-            auto doc_obj = Json::requireObject(doc);
-            auto ver = Modrinth::loadIndexedPackVersion(doc_obj, {}, m_mod.fileinfo().fileName());
-
-            // Minimal IndexedPack to create the metadata
-            ModPlatform::IndexedPack pack;
-            pack.name = m_mod.name();
-            pack.provider = ModPlatform::Provider::MODRINTH;
-            pack.addonId = ver.addonId;
-
-            // Prevent file name mismatch
-            ver.fileName = m_mod.fileinfo().fileName();
-
-            QDir tmp_index_dir(m_index_dir);
-
-            {
-                LocalModUpdateTask update_metadata(m_index_dir, pack, ver);
-                QEventLoop loop;
-                QTimer timeout;
-
-                QObject::connect(&update_metadata, &Task::finished, &loop, &QEventLoop::quit);
-                QObject::connect(&timeout, &QTimer::timeout, &loop, &QEventLoop::quit);
-
-                update_metadata.start();
-                timeout.start(100);
-
-                loop.exec();
-            }
-
-            auto mod_name = m_mod.name();
-            auto meta = new Metadata::ModStruct(Metadata::get(tmp_index_dir, mod_name));
-            m_mod.setMetadata(meta);
         });
 
-        tsk.addTask(ver_task);
+        m_current_task = project_task.get();
-    }
+        project_task->start();
+    });
+
+    connect(version_task.get(), &Task::finished, [=] {
+        version_task->deleteLater();
+        m_current_task = nullptr;
+    });
+
+    if (m_mods.size() > 1)
+        setStatus(tr("Requesting metadata information from %1...").arg(ProviderCaps.readableName(m_provider)));
+    else if (!m_mods.empty())
+        setStatus(tr("Requesting metadata information from %1 for '%2'...")
+                      .arg(ProviderCaps.readableName(m_provider), m_mods.begin().value().name()));
+
+    m_current_task = version_task.get();
+    version_task->start();
 }
 
-void EnsureMetadataTask::flameEnsureMetadata(SequentialTask& tsk, QByteArray& jar_data)
+void EnsureMetadataTask::emitReady(Mod& m)
 {
-    QByteArray jar_data_treated;
+    qDebug() << QString("Generated metadata for %1").arg(m.name());
-    for (char c : jar_data) {
+    emit metadataReady(m);
-        // CF-specific
+
-        if (!(c == 9 || c == 10 || c == 13 || c == 32))
+    m_mods.remove(getHash(m));
-            jar_data_treated.push_back(c);
+}
-    }
+
+void EnsureMetadataTask::emitFail(Mod& m)
+{
+    qDebug() << QString("Failed to generate metadata for %1").arg(m.name());
+    emit metadataFailed(m);
+
+    m_mods.remove(getHash(m));
+}
+
+// Modrinth
+
+NetJob::Ptr EnsureMetadataTask::modrinthVersionsTask()
+{
+    auto hash_type = ProviderCaps.hashType(ModPlatform::Provider::MODRINTH).first();
 
     auto* response = new QByteArray();
+    auto ver_task = modrinth_api.currentVersions(m_mods.keys(), hash_type, response);
 
-    std::list<uint> fingerprints;
+    // Prevents unfortunate timings when aborting the task
-    auto murmur = MurmurHash2(jar_data_treated, jar_data_treated.length());
+    if (!ver_task)
-    fingerprints.push_back(murmur);
+        return {};
-
-    auto ver_task = flame_api.matchFingerprints(fingerprints, response);
-
-    connect(ver_task.get(), &Task::succeeded, this, [this, ver_task, response] {
-        QDir tmp_index_dir(m_index_dir);
 
+    connect(ver_task.get(), &NetJob::succeeded, this, [this, response] {
         QJsonParseError parse_error{};
         QJsonDocument doc = QJsonDocument::fromJson(*response, &parse_error);
         if (parse_error.error != QJsonParseError::NoError) {
-            qWarning() << "Error while parsing JSON response from " << m_mod.name() << " at " << parse_error.offset
+            qWarning() << "Error while parsing JSON response from Modrinth::CurrentVersions at " << parse_error.offset
                        << " reason: " << parse_error.errorString();
             qWarning() << *response;
 
-            ver_task->failed(parse_error.errorString());
+            failed(parse_error.errorString());
+            return;
+        }
+
+        try {
+            auto entries = Json::requireObject(doc);
+            for (auto& hash : m_mods.keys()) {
+                auto mod = m_mods.find(hash).value();
+                try {
+                    auto entry = Json::requireObject(entries, hash);
+
+                    setStatus(tr("Parsing API response from Modrinth for '%1'...").arg(mod.name()));
+                    qDebug() << "Getting version for" << mod.name() << "from Modrinth";
+
+                    m_temp_versions.insert(hash, Modrinth::loadIndexedPackVersion(entry));
+                } catch (Json::JsonException& e) {
+                    qDebug() << e.cause();
+                    qDebug() << entries;
+
+                    emitFail(mod);
+                }
+            }
+        } catch (Json::JsonException& e) {
+            qDebug() << e.cause();
+            qDebug() << doc;
+        }
+    });
+
+    return ver_task;
+}
+
+NetJob::Ptr EnsureMetadataTask::modrinthProjectsTask()
+{
+    QHash<QString, QString> addonIds;
+    for (auto const& data : m_temp_versions)
+        addonIds.insert(data.addonId.toString(), data.hash);
+
+    auto response = new QByteArray();
+    NetJob::Ptr proj_task;
+
+    if (addonIds.isEmpty()) {
+        qWarning() << "No addonId found!";
+    } else if (addonIds.size() == 1) {
+        proj_task = modrinth_api.getProject(*addonIds.keyBegin(), response);
+    } else {
+        proj_task = modrinth_api.getProjects(addonIds.keys(), response);
+    }
+
+    // Prevents unfortunate timings when aborting the task
+    if (!proj_task)
+        return {};
+
+    connect(proj_task.get(), &NetJob::succeeded, this, [this, response, addonIds] {
+        QJsonParseError parse_error{};
+        auto doc = QJsonDocument::fromJson(*response, &parse_error);
+        if (parse_error.error != QJsonParseError::NoError) {
+            qWarning() << "Error while parsing JSON response from Modrinth projects task at " << parse_error.offset
+                       << " reason: " << parse_error.errorString();
+            qWarning() << *response;
+            return;
+        }
+
+        try {
+            QJsonArray entries;
+            if (addonIds.size() == 1)
+                entries = { doc.object() };
+            else
+                entries = Json::requireArray(doc);
+
+            for (auto entry : entries) {
+                auto entry_obj = Json::requireObject(entry);
+                auto entry_id = Json::requireString(entry_obj, "id");
+
+                auto hash = addonIds.find(entry_id).value();
+
+                auto mod = m_mods.find(hash).value();
+
+                try {
+                    setStatus(tr("Parsing API response from Modrinth for '%1'...").arg(mod.name()));
+
+                    ModPlatform::IndexedPack pack;
+                    Modrinth::loadIndexedPack(pack, entry_obj);
+
+                    modrinthCallback(pack, m_temp_versions.find(hash).value(), mod);
+                } catch (Json::JsonException& e) {
+                    qDebug() << e.cause();
+                    qDebug() << entries;
+
+                    emitFail(mod);
+                }
+            }
+        } catch (Json::JsonException& e) {
+            qDebug() << e.cause();
+            qDebug() << doc;
+        }
+    });
+
+    return proj_task;
+}
+
+// Flame
+NetJob::Ptr EnsureMetadataTask::flameVersionsTask()
+{
+    auto* response = new QByteArray();
+
+    std::list<uint> fingerprints;
+    for (auto& murmur : m_mods.keys()) {
+        fingerprints.push_back(murmur.toUInt());
+    }
+
+    auto ver_task = flame_api.matchFingerprints(fingerprints, response);
+
+    connect(ver_task.get(), &Task::succeeded, this, [this, response] {
+        QJsonParseError parse_error{};
+        QJsonDocument doc = QJsonDocument::fromJson(*response, &parse_error);
+        if (parse_error.error != QJsonParseError::NoError) {
+            qWarning() << "Error while parsing JSON response from Modrinth::CurrentVersions at " << parse_error.offset
+                       << " reason: " << parse_error.errorString();
+            qWarning() << *response;
+
+            failed(parse_error.errorString());
             return;
         }
 
         try {
             auto doc_obj = Json::requireObject(doc);
-            auto data_obj = Json::ensureObject(doc_obj, "data");
+            auto data_obj = Json::requireObject(doc_obj, "data");
-            auto match_obj = Json::ensureObject(Json::ensureArray(data_obj, "exactMatches")[0], {});
+            auto data_arr = Json::requireArray(data_obj, "exactMatches");
-            if (match_obj.isEmpty()) {
+
-                qCritical() << "Fingerprint match is empty!";
+            if (data_arr.isEmpty()) {
+                qWarning() << "No matches found for fingerprint search!";
 
-                ver_task->failed(parse_error.errorString());
                 return;
             }
 
-            auto file_obj = Json::ensureObject(match_obj, "file");
+            for (auto match : data_arr) {
+                auto match_obj = Json::ensureObject(match, {});
+                auto file_obj = Json::ensureObject(match_obj, "file", {});
 
-            ModPlatform::IndexedPack pack;
+                if (match_obj.isEmpty() || file_obj.isEmpty()) {
-            pack.name = m_mod.name();
+                    qWarning() << "Fingerprint match is empty!";
-            pack.provider = ModPlatform::Provider::FLAME;
-            pack.addonId = Json::requireInteger(file_obj, "modId");
 
-            ModPlatform::IndexedVersion ver = FlameMod::loadIndexedPackVersion(file_obj);
+                    return;
+                }
 
-            // Prevent file name mismatch
+                auto fingerprint = QString::number(Json::ensureVariant(file_obj, "fileFingerprint").toUInt());
-            ver.fileName = m_mod.fileinfo().fileName();
+                auto mod = m_mods.find(fingerprint);
+                if (mod == m_mods.end()) {
+                    qWarning() << "Invalid fingerprint from the API response.";
+                    continue;
+                }
 
-            {
+                setStatus(tr("Parsing API response from CurseForge for '%1'...").arg(mod->name()));
-                LocalModUpdateTask update_metadata(m_index_dir, pack, ver);
-                QEventLoop loop;
-                QTimer timeout;
 
-                QObject::connect(&update_metadata, &Task::finished, &loop, &QEventLoop::quit);
+                m_temp_versions.insert(fingerprint, FlameMod::loadIndexedPackVersion(file_obj));
-                QObject::connect(&timeout, &QTimer::timeout, &loop, &QEventLoop::quit);
-
-                update_metadata.start();
-                timeout.start(100);
-
-                loop.exec();
             }
 
-            auto mod_name = m_mod.name();
-            auto meta = new Metadata::ModStruct(Metadata::get(tmp_index_dir, mod_name));
-            m_mod.setMetadata(meta);
-
         } catch (Json::JsonException& e) {
-            emitFailed(e.cause() + " : " + e.what());
+            qDebug() << e.cause();
+            qDebug() << doc;
         }
     });
 
-    tsk.addTask(ver_task);
+    return ver_task;
+}
+
+NetJob::Ptr EnsureMetadataTask::flameProjectsTask()
+{
+    QHash<QString, QString> addonIds;
+    for (auto const& hash : m_mods.keys()) {
+        if (m_temp_versions.contains(hash)) {
+            auto const& data = m_temp_versions.find(hash).value();
+            addonIds.insert(data.addonId.toString(), hash);
+        }
+    }
+
+    auto response = new QByteArray();
+    NetJob::Ptr proj_task;
+
+    if (addonIds.isEmpty()) {
+        qWarning() << "No addonId found!";
+    } else if (addonIds.size() == 1) {
+        proj_task = flame_api.getProject(*addonIds.keyBegin(), response);
+    } else {
+        proj_task = flame_api.getProjects(addonIds.keys(), response);
+    }
+
+    // Prevents unfortunate timings when aborting the task
+    if (!proj_task)
+        return {};
+
+    connect(proj_task.get(), &NetJob::succeeded, this, [this, response, addonIds] {
+        QJsonParseError parse_error{};
+        auto doc = QJsonDocument::fromJson(*response, &parse_error);
+        if (parse_error.error != QJsonParseError::NoError) {
+            qWarning() << "Error while parsing JSON response from Modrinth projects task at " << parse_error.offset
+                       << " reason: " << parse_error.errorString();
+            qWarning() << *response;
+            return;
+        }
+
+        try {
+            QJsonArray entries;
+            if (addonIds.size() == 1)
+                entries = { Json::requireObject(Json::requireObject(doc), "data") };
+            else
+                entries = Json::requireArray(Json::requireObject(doc), "data");
+
+            for (auto entry : entries) {
+                auto entry_obj = Json::requireObject(entry);
+
+                auto id = QString::number(Json::requireInteger(entry_obj, "id"));
+                auto hash = addonIds.find(id).value();
+                auto mod = m_mods.find(hash).value();
+
+                try {
+                    setStatus(tr("Parsing API response from CurseForge for '%1'...").arg(mod.name()));
+
+                    ModPlatform::IndexedPack pack;
+                    FlameMod::loadIndexedPack(pack, entry_obj);
+
+                    flameCallback(pack, m_temp_versions.find(hash).value(), mod);
+                } catch (Json::JsonException& e) {
+                    qDebug() << e.cause();
+                    qDebug() << entries;
+
+                    emitFail(mod);
+                }
+            }
+        } catch (Json::JsonException& e) {
+            qDebug() << e.cause();
+            qDebug() << doc;
+        }
+    });
+
+    return proj_task;
+}
+
+void EnsureMetadataTask::modrinthCallback(ModPlatform::IndexedPack& pack, ModPlatform::IndexedVersion& ver, Mod& mod)
+{
+    // Prevent file name mismatch
+    ver.fileName = mod.fileinfo().fileName();
+
+    QDir tmp_index_dir(m_index_dir);
+
+    {
+        LocalModUpdateTask update_metadata(m_index_dir, pack, ver);
+        QEventLoop loop;
+
+        QObject::connect(&update_metadata, &Task::finished, &loop, &QEventLoop::quit);
+
+        update_metadata.start();
+
+        if (!update_metadata.isFinished())
+            loop.exec();
+    }
+
+    auto metadata = Metadata::get(tmp_index_dir, pack.slug);
+    if (!metadata.isValid()) {
+        qCritical() << "Failed to generate metadata at last step!";
+        emitFail(mod);
+        return;
+    }
+
+    mod.setMetadata(metadata);
+
+    emitReady(mod);
+}
+
+void EnsureMetadataTask::flameCallback(ModPlatform::IndexedPack& pack, ModPlatform::IndexedVersion& ver, Mod& mod)
+{
+    try {
+        // Prevent file name mismatch
+        ver.fileName = mod.fileinfo().fileName();
+
+        QDir tmp_index_dir(m_index_dir);
+
+        {
+            LocalModUpdateTask update_metadata(m_index_dir, pack, ver);
+            QEventLoop loop;
+
+            QObject::connect(&update_metadata, &Task::finished, &loop, &QEventLoop::quit);
+
+            update_metadata.start();
+
+            if (!update_metadata.isFinished())
+                loop.exec();
+        }
+
+        auto metadata = Metadata::get(tmp_index_dir, pack.slug);
+        if (!metadata.isValid()) {
+            qCritical() << "Failed to generate metadata at last step!";
+            emitFail(mod);
+            return;
+        }
+
+        mod.setMetadata(metadata);
+
+        emitReady(mod);
+    } catch (Json::JsonException& e) {
+        qDebug() << e.cause();
+
+        emitFail(mod);
+    }
 }

launcher/modplatform/EnsureMetadataTask.h

@@ -2,6 +2,7 @@
 
 #include "ModIndex.h"
 #include "tasks/SequentialTask.h"
+#include "net/NetJob.h"
 
 class Mod;
 class QDir;
@@ -11,7 +12,10 @@ class EnsureMetadataTask : public Task {
     Q_OBJECT
 
    public:
-    EnsureMetadataTask(Mod&, QDir&, bool try_all, ModPlatform::Provider = ModPlatform::Provider::MODRINTH);
+    EnsureMetadataTask(Mod&, QDir, ModPlatform::Provider = ModPlatform::Provider::MODRINTH);
+    EnsureMetadataTask(std::list<Mod>&, QDir, ModPlatform::Provider = ModPlatform::Provider::MODRINTH);
+
+    ~EnsureMetadataTask() = default;
 
    public slots:
     bool abort() override;
@@ -20,22 +24,31 @@ class EnsureMetadataTask : public Task {
 
    private:
     // FIXME: Move to their own namespace
-    void modrinthEnsureMetadata(SequentialTask&, QByteArray&);
+    auto modrinthVersionsTask() -> NetJob::Ptr;
-    void flameEnsureMetadata(SequentialTask&, QByteArray&);
+    auto modrinthProjectsTask() -> NetJob::Ptr;
+
+    auto flameVersionsTask() -> NetJob::Ptr;
+    auto flameProjectsTask() -> NetJob::Ptr;
 
     // Helpers
-    void emitReady();
+    void emitReady(Mod&);
-    void emitFail();
+    void emitFail(Mod&);
+
+    auto getHash(Mod&) -> QString;
+
+   private slots:
+    void modrinthCallback(ModPlatform::IndexedPack& pack, ModPlatform::IndexedVersion& ver, Mod&);
+    void flameCallback(ModPlatform::IndexedPack& pack, ModPlatform::IndexedVersion& ver, Mod&);
 
    signals:
-    void metadataReady();
+    void metadataReady(Mod&);
-    void metadataFailed();
+    void metadataFailed(Mod&);
 
    private:
-    Mod& m_mod;
+    QHash<QString, Mod> m_mods;
-    QDir& m_index_dir;
+    QDir m_index_dir;
     ModPlatform::Provider m_provider;
-    bool m_try_all;
 
-    MultipleOptionsTask* m_task_handler = nullptr;
+    QHash<QString, ModPlatform::IndexedVersion> m_temp_versions;
+    NetJob* m_current_task;
 };

launcher/modplatform/flame/FlameCheckUpdate.cpp

@@ -10,7 +10,6 @@
 #include "ModDownloadTask.h"
 
 static FlameAPI api;
-static ModPlatform::ProviderCapabilities ProviderCaps;
 
 bool FlameCheckUpdate::abort()
 {
@@ -64,6 +63,50 @@ ModPlatform::IndexedPack getProjectInfo(ModPlatform::IndexedVersion& ver_info)
     return pack;
 }
 
+ModPlatform::IndexedVersion getFileInfo(int addonId, int fileId)
+{
+    ModPlatform::IndexedVersion ver;
+
+    QEventLoop loop;
+
+    auto get_file_info_job = new NetJob("Flame::GetFileInfoJob", APPLICATION->network());
+
+    auto response = new QByteArray();
+    auto url = QString("https://api.curseforge.com/v1/mods/%1/files/%2").arg(QString::number(addonId), QString::number(fileId));
+    auto dl = Net::Download::makeByteArray(url, response);
+    get_file_info_job->addNetAction(dl);
+
+    QObject::connect(get_file_info_job, &NetJob::succeeded, [response, &ver]() {
+        QJsonParseError parse_error{};
+        QJsonDocument doc = QJsonDocument::fromJson(*response, &parse_error);
+        if (parse_error.error != QJsonParseError::NoError) {
+            qWarning() << "Error while parsing JSON response from FlameCheckUpdate at " << parse_error.offset
+                       << " reason: " << parse_error.errorString();
+            qWarning() << *response;
+            return;
+        }
+
+        try {
+            auto doc_obj = Json::requireObject(doc);
+            auto data_obj = Json::requireObject(doc_obj, "data");
+            ver = FlameMod::loadIndexedPackVersion(data_obj);
+        } catch (Json::JsonException& e) {
+            qWarning() << e.cause();
+            qDebug() << doc;
+        }
+    });
+
+    QObject::connect(get_file_info_job, &NetJob::finished, [&loop, get_file_info_job] {
+        get_file_info_job->deleteLater();
+        loop.quit();
+    });
+
+    get_file_info_job->start();
+    loop.exec();
+
+    return ver;
+}
+
 /* Check for update:
  * - Get latest version available
  * - Compare hash of the latest version with the current hash
@@ -72,148 +115,60 @@ ModPlatform::IndexedPack getProjectInfo(ModPlatform::IndexedVersion& ver_info)
 void FlameCheckUpdate::executeTask()
 {
     setStatus(tr("Preparing mods for CurseForge..."));
-    setProgress(0, 5);
 
-    QHash<int, Mod> mappings;
+    int i = 0;
-
-    // Create all hashes
-    std::list<uint> murmur_hashes;
-
-    auto best_hash_type = ProviderCaps.hashType(ModPlatform::Provider::FLAME).first();
     for (auto mod : m_mods) {
-        QByteArray jar_data;
+        setStatus(tr("Getting API response from CurseForge for '%1'").arg(mod.name()));
+        setProgress(i++, m_mods.size());
 
-        try {
+        auto latest_ver = api.getLatestVersion({ mod.metadata()->project_id.toString(), m_game_versions, m_loaders });
-            jar_data = FS::read(mod.fileinfo().absoluteFilePath());
-        } catch (FS::FileSystemException& e) {
-            qCritical() << QString("Failed to open / read JAR file of %1").arg(mod.name());
-            qCritical() << QString("Reason: ") << e.cause();
 
-            failed(e.what());
+        // Check if we were aborted while getting the latest version
+        if (m_was_aborted) {
+            aborted();
             return;
         }
 
-        QByteArray jar_data_treated;
+        setStatus(tr("Parsing the API response from CurseForge for '%1'...").arg(mod.name()));
-        for (char c : jar_data) {
+
-            // CF-specific
+        if (!latest_ver.addonId.isValid()) {
-            if (!(c == 9 || c == 10 || c == 13 || c == 32))
+            emit checkFailed(mod, tr("No valid version found for this mod. It's probably unavailable for the current game "
-                jar_data_treated.push_back(c);
+                                     "version / mod loader."));
+            continue;
         }
 
-        auto murmur_hash = MurmurHash2(jar_data_treated, jar_data_treated.length());
+        if (latest_ver.downloadUrl.isEmpty() && latest_ver.fileId != mod.metadata()->file_id) {
-        murmur_hashes.emplace_back(murmur_hash);
+            auto pack = getProjectInfo(latest_ver);
+            auto recover_url = QString("%1/download/%2").arg(pack.websiteUrl, latest_ver.fileId.toString());
+            emit checkFailed(mod, tr("Mod has a new update available, but is opted-out on CurseForge"), recover_url);
 
-        mappings.insert(mod.metadata()->mod_id().toInt(), mod);
+            continue;
-    }
-
-    auto* response = new QByteArray();
-    auto job = api.matchFingerprints(murmur_hashes, response);
-
-    QEventLoop lock;
-
-    connect(job.get(), &Task::succeeded, this, [this, response, &mappings] {
-        QJsonParseError parse_error{};
-        QJsonDocument doc = QJsonDocument::fromJson(*response, &parse_error);
-        if (parse_error.error != QJsonParseError::NoError) {
-            qWarning() << "Error while parsing JSON response from FlameCheckUpdate at " << parse_error.offset
-                       << " reason: " << parse_error.errorString();
-            qWarning() << *response;
-
-            failed(parse_error.errorString());
-            return;
         }
 
-        setStatus(tr("Parsing the first API response from CurseForge..."));
+        if (!latest_ver.hash.isEmpty() && (mod.metadata()->hash != latest_ver.hash || mod.status() == ModStatus::NotInstalled)) {
-        setProgress(2, 5);
+            // Fake pack with the necessary info to pass to the download task :)
+            ModPlatform::IndexedPack pack;
+            pack.name = mod.name();
+            pack.slug = mod.metadata()->slug;
+            pack.addonId = mod.metadata()->project_id;
+            pack.websiteUrl = mod.homeurl();
+            for (auto& author : mod.authors())
+                pack.authors.append({ author });
+            pack.description = mod.description();
+            pack.provider = ModPlatform::Provider::FLAME;
 
-        try {
+            auto old_version = mod.version();
-            auto doc_obj = Json::requireObject(doc);
+            if (old_version.isEmpty() && mod.status() != ModStatus::NotInstalled) {
-            auto data_obj = Json::ensureObject(doc_obj, "data");
+                auto current_ver = getFileInfo(latest_ver.addonId.toInt(), mod.metadata()->file_id.toInt());
-            auto match_arr = Json::ensureArray(data_obj, "exactMatches");
+                old_version = current_ver.version;
-            for (auto match : match_arr) {
-                auto match_obj = Json::ensureObject(match);
-
-                ModPlatform::IndexedVersion current_ver;
-                try {
-                    auto file_obj = Json::requireObject(match_obj, "file");
-                    current_ver = FlameMod::loadIndexedPackVersion(file_obj);
-                } catch (Json::JsonException& e) {
-                    qCritical() << "Error while parsing Flame indexed version";
-                    qCritical() << e.what();
-                    failed(tr("An error occured while parsing a CurseForge indexed version!"));
-                    return;
-                }
-
-                auto mod_iter = mappings.find(current_ver.addonId.toInt());
-                if (mod_iter == mappings.end()) {
-                    qCritical() << "Failed to remap mod from Flame!";
-                    qDebug() << match_obj;
-                    continue;
-                }
-
-                auto mod = mod_iter.value();
-
-                setStatus(tr("Waiting for the API response from CurseForge for '%1'...").arg(mod.name()));
-                setProgress(3, 5);
-
-                auto latest_ver = api.getLatestVersion({ current_ver.addonId.toString(), m_game_versions, m_loaders });
-
-                // Check if we were aborted while getting the latest version
-                if (m_was_aborted) {
-                    aborted();
-                    return;
-                }
-
-                setStatus(tr("Parsing the API response from CurseForge for '%1'...").arg(mod.name()));
-                setProgress(4, 5);
-
-                if (!latest_ver.addonId.isValid()) {
-                    emit checkFailed(
-                        mod,
-                        tr("No valid version found for this mod. It's probably unavailable for the current game version / mod loader."));
-                    continue;
-                }
-
-                if (latest_ver.downloadUrl.isEmpty() && latest_ver.fileId != current_ver.fileId) {
-                    auto pack = getProjectInfo(latest_ver);
-                    auto recover_url = QString("%1/download/%2").arg(pack.websiteUrl, latest_ver.fileId.toString());
-                    emit checkFailed(mod, tr("Mod has a new update available, but is opted-out on CurseForge"), recover_url);
-
-                    continue;
-                }
-
-                if (!latest_ver.hash.isEmpty() && (current_ver.hash != latest_ver.hash || mod.status() == ModStatus::NotInstalled)) {
-                    // Fake pack with the necessary info to pass to the download task :)
-                    ModPlatform::IndexedPack pack;
-                    pack.name = mod.name();
-                    pack.addonId = mod.metadata()->project_id;
-                    pack.websiteUrl = mod.homeurl();
-                    for (auto& author : mod.authors())
-                        pack.authors.append({ author });
-                    pack.description = mod.description();
-                    pack.provider = ModPlatform::Provider::FLAME;
-
-                    auto download_task = new ModDownloadTask(pack, latest_ver, m_mods_folder);
-                    m_updatable.emplace_back(mod.name(), current_ver.hash, current_ver.version, latest_ver.version,
-                                             api.getModFileChangelog(latest_ver.addonId.toInt(), latest_ver.fileId.toInt()),
-                                             ModPlatform::Provider::FLAME, download_task);
-                }
             }
 
-        } catch (Json::JsonException& e) {
+            auto download_task = new ModDownloadTask(pack, latest_ver, m_mods_folder);
-            failed(e.cause() + " : " + e.what());
+            m_updatable.emplace_back(mod.name(), mod.metadata()->hash, old_version, latest_ver.version,
+                                     api.getModFileChangelog(latest_ver.addonId.toInt(), latest_ver.fileId.toInt()),
+                                     ModPlatform::Provider::FLAME, download_task);
         }
-    });
+    }
-
-    connect(job.get(), &Task::finished, &lock, &QEventLoop::quit);
-
-    setStatus(tr("Waiting for the first API response from CurseForge..."));
-    setProgress(1, 5);
-
-    m_net_job = job.get();
-    job->start();
-
-    lock.exec();
 
     emitSucceeded();
 }

launcher/modplatform/modrinth/ModrinthCheckUpdate.cpp

@@ -88,7 +88,9 @@ void ModrinthCheckUpdate::executeTask()
                     qDebug() << "Mod " << mappings.find(hash).value().name() << " got an empty response.";
                     qDebug() << "Hash: " << hash;
 
-                    emit checkFailed(mappings.find(hash).value(), tr("Couldn't find the latest version of this mod with the correct mod loader and game version."));
+                    emit checkFailed(
+                        mappings.find(hash).value(),
+                        tr("No valid version found for this mod. It's probably unavailable for the current game version / mod loader."));
 
                     continue;
                 }
@@ -134,6 +136,7 @@ void ModrinthCheckUpdate::executeTask()
                     // Fake pack with the necessary info to pass to the download task :)
                     ModPlatform::IndexedPack pack;
                     pack.name = mod.name();
+                    pack.slug = mod.metadata()->slug;
                     pack.addonId = mod.metadata()->project_id;
                     pack.websiteUrl = mod.homeurl();
                     for (auto& author : mod.authors())

launcher/ui/dialogs/ModUpdateDialog.cpp

@@ -34,12 +34,13 @@ static ModAPI::ModLoaderTypes mcLoaders(BaseInstance* inst)
 
 ModUpdateDialog::ModUpdateDialog(QWidget* parent,
                                  BaseInstance* instance,
-                                 const std::shared_ptr<ModFolderModel>& mods,
+                                 const std::shared_ptr<ModFolderModel> mods,
                                  std::list<Mod>& search_for)
     : ReviewMessageBox(parent, tr("Confirm mods to update"), "")
     , m_parent(parent)
     , m_mod_model(mods)
     , m_candidates(search_for)
+    , m_second_try_metadata(new SequentialTask())
     , m_instance(instance)
 {
     ReviewMessageBox::setGeometry(0, 0, 800, 600);
@@ -47,15 +48,6 @@ ModUpdateDialog::ModUpdateDialog(QWidget* parent,
     ui->explainLabel->setText(tr("You're about to update the following mods:"));
     ui->onlyCheckedLabel->setText(tr("Only mods with a check will be updated!"));
 
-    connect(&m_check_task, &Task::failed, this,
-            [&](QString reason) { CustomMessageBox::selectable(this, tr("Error"), reason, QMessageBox::Critical)->exec(); });
-
-    connect(&m_check_task, &Task::succeeded, this, [&]() {
-        QStringList warnings = m_check_task.warnings();
-        if (warnings.count()) {
-            CustomMessageBox::selectable(this, tr("Warnings"), warnings.join('\n'), QMessageBox::Warning)->exec();
-        }
-    });
 }
 
 void ModUpdateDialog::checkCandidates()
@@ -89,26 +81,39 @@ void ModUpdateDialog::checkCandidates()
     auto versions = mcVersions(m_instance);
     auto loaders = mcLoaders(m_instance);
 
+    SequentialTask check_task (m_parent, tr("Checking for updates"));
+
     if (!m_modrinth_to_update.empty()) {
         m_modrinth_check_task = new ModrinthCheckUpdate(m_modrinth_to_update, versions, loaders, m_mod_model);
         connect(m_modrinth_check_task, &CheckUpdateTask::checkFailed, this,
                 [this](Mod mod, QString reason, QUrl recover_url) { m_failed_check_update.emplace_back(mod, reason, recover_url); });
-        m_check_task.addTask(m_modrinth_check_task);
+        check_task.addTask(m_modrinth_check_task);
     }
 
     if (!m_flame_to_update.empty()) {
         m_flame_check_task = new FlameCheckUpdate(m_flame_to_update, versions, loaders, m_mod_model);
         connect(m_flame_check_task, &CheckUpdateTask::checkFailed, this,
                 [this](Mod mod, QString reason, QUrl recover_url) { m_failed_check_update.emplace_back(mod, reason, recover_url); });
-        m_check_task.addTask(m_flame_check_task);
+        check_task.addTask(m_flame_check_task);
     }
 
+    connect(&check_task, &Task::failed, this,
+            [&](QString reason) { CustomMessageBox::selectable(this, tr("Error"), reason, QMessageBox::Critical)->exec(); });
+
+    connect(&check_task, &Task::succeeded, this, [&]() {
+        QStringList warnings = check_task.warnings();
+        if (warnings.count()) {
+            CustomMessageBox::selectable(this, tr("Warnings"), warnings.join('\n'), QMessageBox::Warning)->exec();
+        }
+    });
+
     // Check for updates
+    // FIXME: SOMEHOW THIS IS NOT MODAL???????
     ProgressDialog progress_dialog(m_parent);
     progress_dialog.setSkipButton(true, tr("Abort"));
     progress_dialog.setVisible(true);
     progress_dialog.setWindowTitle(tr("Checking for updates..."));
-    auto ret = progress_dialog.execWithTask(&m_check_task);
+    auto ret = progress_dialog.execWithTask(&check_task);
 
     // If the dialog was skipped / some download error happened
     if (ret == QDialog::DialogCode::Rejected) {
@@ -183,13 +188,29 @@ auto ModUpdateDialog::ensureMetadata() -> bool
 {
     auto index_dir = indexDir();
 
-    auto* seq = new SequentialTask(m_parent, tr("Looking for metadata"));
+    SequentialTask seq(m_parent, tr("Looking for metadata"));
+
+    // A better use of data structures here could remove the need for this QHash
+    QHash<QString, bool> should_try_others;
+    std::list<Mod> modrinth_tmp;
+    std::list<Mod> flame_tmp;
 
     bool confirm_rest = false;
     bool try_others_rest = false;
     bool skip_rest = false;
     ModPlatform::Provider provider_rest = ModPlatform::Provider::MODRINTH;
 
+    auto addToTmp = [&](Mod& m, ModPlatform::Provider p) {
+        switch (p) {
+            case ModPlatform::Provider::MODRINTH:
+                modrinth_tmp.push_back(m);
+                break;
+            case ModPlatform::Provider::FLAME:
+                flame_tmp.push_back(m);
+                break;
+        }
+    };
+
     for (auto& candidate : m_candidates) {
         if (candidate.status() != ModStatus::NoMetadata) {
             onMetadataEnsured(candidate);
@@ -200,10 +221,8 @@ auto ModUpdateDialog::ensureMetadata() -> bool
             continue;
 
         if (confirm_rest) {
-            auto* task = new EnsureMetadataTask(candidate, index_dir, try_others_rest, provider_rest);
+            addToTmp(candidate, provider_rest);
-            connect(task, &EnsureMetadataTask::metadataReady, [this, &candidate] { onMetadataEnsured(candidate); });
+            should_try_others.insert(candidate.internal_id(), try_others_rest);
-            connect(task, &EnsureMetadataTask::metadataFailed, [this, &candidate] { onMetadataFailed(candidate); });
-            seq->addTask(task);
             continue;
         }
 
@@ -224,18 +243,36 @@ auto ModUpdateDialog::ensureMetadata() -> bool
             try_others_rest = response.try_others;
         }
 
-        if (confirmed) {
+        should_try_others.insert(candidate.internal_id(), response.try_others);
-            auto* task = new EnsureMetadataTask(candidate, index_dir, response.try_others, response.chosen);
+
-            connect(task, &EnsureMetadataTask::metadataReady, [this, &candidate] { onMetadataEnsured(candidate); });
+        if (confirmed)
-            connect(task, &EnsureMetadataTask::metadataFailed, [this, &candidate] { onMetadataFailed(candidate); });
+            addToTmp(candidate, response.chosen);
-            seq->addTask(task);
-        }
     }
 
+    if (!modrinth_tmp.empty()) {
+        auto* modrinth_task = new EnsureMetadataTask(modrinth_tmp, index_dir, ModPlatform::Provider::MODRINTH);
+        connect(modrinth_task, &EnsureMetadataTask::metadataReady, [this](Mod& candidate) { onMetadataEnsured(candidate); });
+        connect(modrinth_task, &EnsureMetadataTask::metadataFailed, [this, &should_try_others](Mod& candidate) {
+            onMetadataFailed(candidate, should_try_others.find(candidate.internal_id()).value(), ModPlatform::Provider::MODRINTH);
+        });
+        seq.addTask(modrinth_task);
+    }
+
+    if (!flame_tmp.empty()) {
+        auto* flame_task = new EnsureMetadataTask(flame_tmp, index_dir, ModPlatform::Provider::FLAME);
+        connect(flame_task, &EnsureMetadataTask::metadataReady, [this](Mod& candidate) { onMetadataEnsured(candidate); });
+        connect(flame_task, &EnsureMetadataTask::metadataFailed, [this, &should_try_others](Mod& candidate) {
+            onMetadataFailed(candidate, should_try_others.find(candidate.internal_id()).value(), ModPlatform::Provider::FLAME);
+        });
+        seq.addTask(flame_task);
+    }
+
+    seq.addTask(m_second_try_metadata);
+
     ProgressDialog checking_dialog(m_parent);
     checking_dialog.setSkipButton(true, tr("Abort"));
     checking_dialog.setWindowTitle(tr("Generating metadata..."));
-    auto ret_metadata = checking_dialog.execWithTask(seq);
+    auto ret_metadata = checking_dialog.execWithTask(&seq);
 
     return (ret_metadata != QDialog::DialogCode::Rejected);
 }
@@ -256,9 +293,31 @@ void ModUpdateDialog::onMetadataEnsured(Mod& mod)
     }
 }
 
-void ModUpdateDialog::onMetadataFailed(Mod& mod)
+ModPlatform::Provider next(ModPlatform::Provider p)
 {
-    m_failed_metadata.push_back(mod);
+    switch (p) {
+        case ModPlatform::Provider::MODRINTH:
+            return ModPlatform::Provider::FLAME;
+        case ModPlatform::Provider::FLAME:
+            return ModPlatform::Provider::MODRINTH;
+    }
+
+    return ModPlatform::Provider::FLAME;
+}
+
+void ModUpdateDialog::onMetadataFailed(Mod& mod, bool try_others, ModPlatform::Provider first_choice)
+{
+    if (try_others) {
+        auto index_dir = indexDir();
+
+        auto* task = new EnsureMetadataTask(mod, index_dir, next(first_choice));
+        connect(task, &EnsureMetadataTask::metadataReady, [this](Mod& candidate) { onMetadataEnsured(candidate); });
+        connect(task, &EnsureMetadataTask::metadataFailed, [this](Mod& candidate) { onMetadataFailed(candidate, false); });
+
+        m_second_try_metadata->addTask(task);
+    } else {
+        m_failed_metadata.push_back(mod);
+    }
 }
 
 void ModUpdateDialog::appendMod(CheckUpdateTask::UpdatableMod const& info)

launcher/ui/dialogs/ModUpdateDialog.h

@@ -17,7 +17,7 @@ class ModUpdateDialog final : public ReviewMessageBox {
    public:
     explicit ModUpdateDialog(QWidget* parent,
                              BaseInstance* instance,
-                             const std::shared_ptr<ModFolderModel>& mod_model,
+                             const std::shared_ptr<ModFolderModel> mod_model,
                              std::list<Mod>& search_for);
 
     void checkCandidates();
@@ -35,21 +35,21 @@ class ModUpdateDialog final : public ReviewMessageBox {
 
    private slots:
     void onMetadataEnsured(Mod&);
-    void onMetadataFailed(Mod&);
+    void onMetadataFailed(Mod&, bool try_others = false, ModPlatform::Provider first_choice = ModPlatform::Provider::MODRINTH);
 
    private:
     QWidget* m_parent;
 
-    SequentialTask m_check_task;
     ModrinthCheckUpdate* m_modrinth_check_task = nullptr;
     FlameCheckUpdate* m_flame_check_task = nullptr;
 
-    const std::shared_ptr<ModFolderModel>& m_mod_model;
+    const std::shared_ptr<ModFolderModel> m_mod_model;
 
     std::list<Mod>& m_candidates;
     std::list<Mod> m_modrinth_to_update;
     std::list<Mod> m_flame_to_update;
 
+    SequentialTask* m_second_try_metadata;
     std::list<Mod> m_failed_metadata;
     std::list<std::tuple<Mod, QString, QUrl>> m_failed_check_update;