From 6a4e06a76dd33c5057f34d0ce6dc0239ba5babbe Mon Sep 17 00:00:00 2001 From: T1ti <40864460+T1ti@users.noreply.github.com> Date: Tue, 23 Sep 2025 05:48:37 +0200 Subject: [PATCH] massively speed up SQL --- src/external/blizzard-database-library | 2 +- src/noggit/database/ClientDatabase.cpp | 65 +++++++++---------- src/noggit/database/ClientDatabase.h | 24 +++---- .../database/clientDbDefinitions/MapDB.h | 44 +++++++++++++ .../Ui/MapCreationWizard.cpp | 10 +++ .../ui/windows/noggitWindow/NoggitWindow.cpp | 6 +- 6 files changed, 101 insertions(+), 50 deletions(-) diff --git a/src/external/blizzard-database-library b/src/external/blizzard-database-library index 33bab9f0..34193192 160000 --- a/src/external/blizzard-database-library +++ b/src/external/blizzard-database-library @@ -1 +1 @@ -Subproject commit 33bab9f08cdcb183b030dd0b24f9d0a47d5b1d01 +Subproject commit 34193192a45ab5b5ff5908b2895652d6bcf63c79 diff --git a/src/noggit/database/ClientDatabase.cpp b/src/noggit/database/ClientDatabase.cpp index d919cb03..77532c05 100644 --- a/src/noggit/database/ClientDatabase.cpp +++ b/src/noggit/database/ClientDatabase.cpp @@ -19,13 +19,15 @@ namespace Noggit }; + void ClientDatabase::setDatabaseMode(DatabaseMode mode) + { + _database_mode = mode; + } + + DatabaseMode ClientDatabase::_database_mode = DatabaseMode::ClientStorage; DatabaseMode ClientDatabase::databaseMode() { - bool setting_use_sql_db = true; // todo QSETTING - QSettings settings; - setting_use_sql_db = settings.value("project/mysql/enabled", false).toBool(); - - return setting_use_sql_db ? DatabaseMode::Sql : DatabaseMode::ClientStorage; + return _database_mode; } ClientDatabaseTable ClientDatabase::getTable(const std::string& tableName) @@ -49,7 +51,7 @@ namespace Noggit // insert if fresh_table, otherwise replace? - auto row_definition = GetRecordDefinition(); + auto& row_definition = GetRecordDefinition(); auto sql_record_format = recordFormat(); auto client_table_iterator = getClientTable().Records(); @@ -184,7 +186,7 @@ namespace Noggit // executes query in client db and checks errors // use isActive to check if it properly ran, not isValid. - QSqlQuery ClientDatabase::executeQuery(const QString& sql) + QSqlQuery ClientDatabase::executeQuery(const QString& sql, bool forward_only) { auto db_mgr = Noggit::Sql::SqlDatabaseManager::instance().noggitDatabase(); QSqlQuery query(Noggit::Sql::SqlDatabaseManager::instance().noggitDatabase()); @@ -192,6 +194,8 @@ namespace Noggit qDebug() << "Executing query : " << sql; QElapsedTimer timer; timer.start(); + if (forward_only) // call this for browsing large data sets + query.setForwardOnly(true); if (!query.exec(sql)) { LogError << "SQL query failed:" << query.lastError().text().toStdString(); @@ -288,7 +292,7 @@ namespace Noggit { auto record_format = std::vector(); - auto row_definition = GetRecordDefinition(); + auto& row_definition = GetRecordDefinition(); for (int col_idx = 0; col_idx < row_definition.ColumnDefinitions.size(); col_idx++) { auto& column_def = row_definition.ColumnDefinitions[col_idx]; @@ -426,9 +430,11 @@ namespace Noggit return table_is_valid; } - Structures::BlizzardDatabaseRow ClientDatabaseTable::sqlRecordToDatabaseRow(const QSqlRecord& record) const + // Structures::BlizzardDatabaseRow ClientDatabaseTable::sqlRecordToDatabaseRow(const QSqlRecord& record) const + Structures::BlizzardDatabaseRow ClientDatabaseTable::sqlRecordToDatabaseRow(QSqlQuery& record) const + { - auto row_definition = GetRecordDefinition(); + auto& row_definition = GetRecordDefinition(); auto database_row = Structures::BlizzardDatabaseRow(-1); @@ -439,18 +445,13 @@ namespace Noggit auto& column_def = row_definition.ColumnDefinitions[column_def_idx]; auto database_column = Structures::BlizzardDatabaseColumn(); - auto value = std::string(); if (column_def.Type == "locstring") { - std::vector localizedValues = std::vector(); + database_column.Values.resize(16); for (int loc_idx = 0; loc_idx < 16; loc_idx++) { - localizedValues.push_back(record.value(field_idx++).toString().toStdString()); + database_column.Values[loc_idx] = (record.value(field_idx++).toString().toStdString()); } - - database_column.Values = localizedValues; - database_row.Columns[column_def.Name] = database_column; - // currently loc mask is set to a separate column because wdbc reader does it. auto loc_mask_column = Structures::BlizzardDatabaseColumn(); loc_mask_column.Value = record.value(field_idx++).toString().toStdString(); @@ -460,21 +461,21 @@ namespace Noggit { if (column_def.arrLength > 1) // array { + database_column.Values.resize(column_def.arrLength); for (int i = 0; i < column_def.arrLength; i++) { - database_column.Values.push_back(record.value(field_idx++).toString().toStdString()); + database_column.Values[i] = (record.value(field_idx++).toString().toStdString()); } } else // single value { - value = record.value(field_idx++).toString().toStdString(); + database_column.Value = record.value(field_idx++).toString().toStdString(); if (column_def.isID) - Id = std::stoi(value); + Id = std::stoi(database_column.Value); } } - database_column.Value = value; - database_row.Columns[column_def.Name] = database_column; + database_row.Columns[column_def.Name] = std::move(database_column); } assert(Id != -1); // no id found database_row.RecordId = Id; @@ -561,7 +562,7 @@ namespace Noggit return std::optional(); }*/ - Structures::BlizzardDatabaseRowDefinition ClientDatabaseTable::GetRecordDefinition() const + Structures::BlizzardDatabaseRowDefinition& ClientDatabaseTable::GetRecordDefinition() const { return Noggit::Project::CurrentProject::get()->ClientDatabase->TableRecordDefinition(_tableName); } @@ -594,9 +595,8 @@ namespace Noggit if (query.next()) { - QSqlRecord record = query.record(); - - auto database_row = sqlRecordToDatabaseRow(record); + // QSqlRecord record = query.record(); // slow af + auto database_row = sqlRecordToDatabaseRow(query); return database_row; } @@ -630,7 +630,7 @@ namespace Noggit { QString sql = QString("SELECT * FROM `%1`").arg(_table.getSqlTableName().c_str()); // ORDER BY ID ? - _query = ClientDatabase::executeQuery(sql); + _query = ClientDatabase::executeQuery(sql, true); _querry_valid = _query.isActive(); // if query.exec ran properly querryAdvance(); @@ -653,7 +653,6 @@ namespace Noggit return _client_iterator.Next(); else { - // if (_query.next()) if (!_querry_valid || !_hasNext) { assert(false); @@ -661,12 +660,12 @@ namespace Noggit } // always store one row in advance to know if it's the last one - auto row = _table.sqlRecordToDatabaseRow(_nextRecord); - assert(row.RecordId != -1); + // auto row = _table.sqlRecordToDatabaseRow(_nextRecord); + assert(_nextRecord.RecordId != -1); querryAdvance(); - return row; + return _nextRecord; } } @@ -675,7 +674,8 @@ namespace Noggit { if (_query.next()) { - _nextRecord = _query.record(); + // _nextRecord = _query.record(); + _nextRecord = _table.sqlRecordToDatabaseRow(_query); _hasNext = true; } else @@ -684,5 +684,4 @@ namespace Noggit } } - } \ No newline at end of file diff --git a/src/noggit/database/ClientDatabase.h b/src/noggit/database/ClientDatabase.h index de8bc614..73eddd91 100644 --- a/src/noggit/database/ClientDatabase.h +++ b/src/noggit/database/ClientDatabase.h @@ -45,12 +45,8 @@ namespace Noggit { friend class ClientDatabaseTable; public: - // static ClientDatabase& instance() - // { - // static ClientDatabase _instance; - // return _instance; - // } + static void setDatabaseMode(DatabaseMode mode); static DatabaseMode databaseMode(); // sql or client storage static ClientDatabaseTable getTable(const std::string& tableName); @@ -61,14 +57,10 @@ namespace Noggit static void TODODeploySqlToClient(); - static QSqlQuery executeQuery(const QString& sql); + static QSqlQuery executeQuery(const QString& sql, bool forward_only = true); // forward onl means can't browse query backward, but massively speeds up forward iteration private: - // ClientDatabase() = default; - // ~ClientDatabase() = default; - // ClientDatabase(const ClientDatabase&) = delete; - // ClientDatabase& operator=(const ClientDatabase&) = delete; - + static DatabaseMode _database_mode; static Structures::BlizzardDatabaseRow clientRowById(const std::string& tableName, unsigned int id); static Structures::BlizzardDatabaseRow sqlRowById(const std::string& tableName, unsigned int id); @@ -88,7 +80,6 @@ namespace Noggit private: const ClientDatabaseTable& _table; - // DatabaseMode _mode; // client BlizzardDatabaseRecordCollection _client_iterator; @@ -98,7 +89,8 @@ namespace Noggit bool _querryHasStarted = false; bool _querry_valid = false; bool _hasNext = false; - QSqlRecord _nextRecord; + // QSqlRecord _nextRecord; + Structures::BlizzardDatabaseRow _nextRecord; void querryAdvance(); }; @@ -111,6 +103,7 @@ namespace Noggit private: const std::string _tableName; const QString _qtTableName; + const Structures::BlizzardDatabaseRowDefinition _row_definition; public: ClientDatabaseTable(std::string tableName); @@ -120,7 +113,7 @@ namespace Noggit unsigned int RecordCount() const; int ColumnCount() const; int getRecordSize() const; - Structures::BlizzardDatabaseRowDefinition GetRecordDefinition() const; + Structures::BlizzardDatabaseRowDefinition& GetRecordDefinition() const; // get rows data std::optional RecordById(unsigned int id) const; @@ -148,7 +141,8 @@ namespace Noggit std::vector recordFormat() const; // true record format for all columns, not array size/loc etc. eg returns all 17 columns for loc. bool createSQLTableIfNotExist(); bool verifySqlTableIntegrity(); - Structures::BlizzardDatabaseRow sqlRecordToDatabaseRow(const QSqlRecord& record) const; + // Structures::BlizzardDatabaseRow sqlRecordToDatabaseRow(const QSqlRecord& record) const; + Structures::BlizzardDatabaseRow sqlRecordToDatabaseRow(QSqlQuery& query) const; }; } diff --git a/src/noggit/database/clientDbDefinitions/MapDB.h b/src/noggit/database/clientDbDefinitions/MapDB.h index e69de29b..e17e8480 100644 --- a/src/noggit/database/clientDbDefinitions/MapDB.h +++ b/src/noggit/database/clientDbDefinitions/MapDB.h @@ -0,0 +1,44 @@ +#pragma once +#include + +// experimental + + +struct MapDbRow : BlizzardDatabaseLib::Structures::BlizzardDatabaseRow +{ + uint AreaType() { return getUInt("InstanceType"); }; // uint +}; + +class MapDb : public Noggit::ClientDatabaseTable +{ +public: + MapDb() : + ClientDatabaseTable("Map") + {} + uint MapID() ; // uint + + std::optional mapRecordById(unsigned int id) { return static_cast(RecordById(id).value()); }; + + /// Fields + static const size_t MapID = 0; // uint + static const size_t InternalName = 1; // string + static const size_t AreaType = 2; // uint + static const size_t Flags = 3; // uint + static const size_t IsBattleground = 4; // uint + static const size_t Name = 5; // loc + static const size_t AreaTableID = 22; // uint + static const size_t MapDescriptionAlliance = 23; // loc + static const size_t MapDescriptionHorde = 40; // loc + static const size_t LoadingScreen = 57; // uint [LoadingScreen] + static const size_t minimapIconScale = 58; // uint [LoadingScreen] + static const size_t corpseMapID = 59; // iRefID Points to column 1, -1 if none + static const size_t corpseX = 60; // Float The X - Coord of the instance entrance + static const size_t corpseY = 61; // Float The Y - Coord of the instance entrance + static const size_t TimeOfDayOverride = 62; // Integer Set to - 1 for everything but Orgrimmar and Dalaran arena.For those, the time of day will change to this. + static const size_t ExpansionID = 63; // Integer Vanilla : 0, BC : 1, WotLK : 2 + static const size_t RaidOffset = 64; // Integer + static const size_t NumberOfPlayers = 65; // Integer Used for reset time? + + static std::string getMapName(int pMapID); + static int findMapName(const std::string& map_name); +}; \ No newline at end of file diff --git a/src/noggit/ui/tools/MapCreationWizard/Ui/MapCreationWizard.cpp b/src/noggit/ui/tools/MapCreationWizard/Ui/MapCreationWizard.cpp index 154559ca..ba13a0fc 100755 --- a/src/noggit/ui/tools/MapCreationWizard/Ui/MapCreationWizard.cpp +++ b/src/noggit/ui/tools/MapCreationWizard/Ui/MapCreationWizard.cpp @@ -93,6 +93,16 @@ MapCreationWizard::MapCreationWizard(std::shared_ptr pro auto testtable = ClientDatabase::getTable("WMOAreaTable"); // auto& testtable = Noggit::Project::CurrentProject::get()->ClientDatabase->LoadTable("WMOAreaTable", readFileAsIMemStream); + // + // for (DBCFile::Iterator i = gWMOAreaTableDB.begin(); i != gWMOAreaTableDB.end(); ++i) + // { + // unsigned int lol = i->getUInt(WMOAreaTableDB::ID); + // for (int ii = 0; ii < WMOAreaTableDB::Name; ii++) + // { + // unsigned int lol = i->getUInt(ii); + // } + // i->getLocalizedString(WMOAreaTableDB::Name); + // } qint64 elapsedMs = timer.elapsed(); Log << "gettable() in : " << elapsedMs << "ms" << std::endl; diff --git a/src/noggit/ui/windows/noggitWindow/NoggitWindow.cpp b/src/noggit/ui/windows/noggitWindow/NoggitWindow.cpp index 4a99b3dc..d882c964 100755 --- a/src/noggit/ui/windows/noggitWindow/NoggitWindow.cpp +++ b/src/noggit/ui/windows/noggitWindow/NoggitWindow.cpp @@ -78,7 +78,8 @@ namespace Noggit::Ui::Windows } else { - LogError << "NoggitWindow() : Unsupported project version, skipping loading DBCs." << std::endl; + assert(false); // TODO + LogError << "NoggitWindow() : Unsupported project version, skipping loading DBCs." << std::endl; } _settings = new settings(this); @@ -86,6 +87,9 @@ namespace Noggit::Ui::Windows QSettings settings; // connect to databases bool use_mysql = settings.value("project/mysql/enabled", false).toBool(); + + ClientDatabase::setDatabaseMode(use_mysql ? DatabaseMode::Sql : DatabaseMode::ClientStorage); + if (use_mysql) { auto host = settings.value("project/mysql/server").toString();