natsirt/noggit-red
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

massively speed up SQL

Browse Source
This commit is contained in:
T1ti
2025-09-23 05:48:37 +02:00
parent 578c83a72c
commit 6a4e06a76d
6 changed files with 101 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/external/blizzard-database-library vendored

Submodule src/external/blizzard-database-library updated: 33bab9f08c...34193192a4

65
src/noggit/database/ClientDatabase.cpp
View File

@@ -19,13 +19,15 @@ namespace Noggit
};
void ClientDatabase::setDatabaseMode(DatabaseMode mode)
{
_database_mode = mode;
}
DatabaseMode ClientDatabase::_database_mode = DatabaseMode::ClientStorage;
DatabaseMode ClientDatabase::databaseMode()
{
bool setting_use_sql_db = true; // todo QSETTING
QSettings settings;
setting_use_sql_db = settings.value("project/mysql/enabled", false).toBool();
return setting_use_sql_db ? DatabaseMode::Sql : DatabaseMode::ClientStorage;
return _database_mode;
}
ClientDatabaseTable ClientDatabase::getTable(const std::string& tableName)
@@ -49,7 +51,7 @@ namespace Noggit
// insert if fresh_table, otherwise replace?
auto row_definition = GetRecordDefinition();
auto& row_definition = GetRecordDefinition();
auto sql_record_format = recordFormat();
auto client_table_iterator = getClientTable().Records();
@@ -184,7 +186,7 @@ namespace Noggit
// executes query in client db and checks errors
// use isActive to check if it properly ran, not isValid.
QSqlQuery ClientDatabase::executeQuery(const QString& sql)
QSqlQuery ClientDatabase::executeQuery(const QString& sql, bool forward_only)
{
auto db_mgr = Noggit::Sql::SqlDatabaseManager::instance().noggitDatabase();
QSqlQuery query(Noggit::Sql::SqlDatabaseManager::instance().noggitDatabase());
@@ -192,6 +194,8 @@ namespace Noggit
qDebug() << "Executing query : " << sql;
QElapsedTimer timer;
timer.start();
if (forward_only) // call this for browsing large data sets
query.setForwardOnly(true);
if (!query.exec(sql))
{
LogError << "SQL query failed:" << query.lastError().text().toStdString();
@@ -288,7 +292,7 @@ namespace Noggit
{
auto record_format = std::vector<DbColumnFormat>();
auto row_definition = GetRecordDefinition();
auto& row_definition = GetRecordDefinition();
for (int col_idx = 0; col_idx < row_definition.ColumnDefinitions.size(); col_idx++)
{
auto& column_def = row_definition.ColumnDefinitions[col_idx];
@@ -426,9 +430,11 @@ namespace Noggit
return table_is_valid;
}
Structures::BlizzardDatabaseRow ClientDatabaseTable::sqlRecordToDatabaseRow(const QSqlRecord& record) const
// Structures::BlizzardDatabaseRow ClientDatabaseTable::sqlRecordToDatabaseRow(const QSqlRecord& record) const
Structures::BlizzardDatabaseRow ClientDatabaseTable::sqlRecordToDatabaseRow(QSqlQuery& record) const
{
auto row_definition = GetRecordDefinition();
auto& row_definition = GetRecordDefinition();
auto database_row = Structures::BlizzardDatabaseRow(-1);
@@ -439,18 +445,13 @@ namespace Noggit
auto& column_def = row_definition.ColumnDefinitions[column_def_idx];
auto database_column = Structures::BlizzardDatabaseColumn();
auto value = std::string();
if (column_def.Type == "locstring")
{
std::vector<std::string> localizedValues = std::vector<std::string>();
database_column.Values.resize(16);
for (int loc_idx = 0; loc_idx < 16; loc_idx++)
{
localizedValues.push_back(record.value(field_idx++).toString().toStdString());
database_column.Values[loc_idx] = (record.value(field_idx++).toString().toStdString());
}
database_column.Values = localizedValues;
database_row.Columns[column_def.Name] = database_column;
// currently loc mask is set to a separate column because wdbc reader does it.
auto loc_mask_column = Structures::BlizzardDatabaseColumn();
loc_mask_column.Value = record.value(field_idx++).toString().toStdString();
@@ -460,21 +461,21 @@ namespace Noggit
{
if (column_def.arrLength > 1) // array
{
database_column.Values.resize(column_def.arrLength);
for (int i = 0; i < column_def.arrLength; i++)
{
database_column.Values.push_back(record.value(field_idx++).toString().toStdString());
database_column.Values[i] = (record.value(field_idx++).toString().toStdString());
}
}
else // single value
{
value = record.value(field_idx++).toString().toStdString();
database_column.Value = record.value(field_idx++).toString().toStdString();
if (column_def.isID)
Id = std::stoi(value);
Id = std::stoi(database_column.Value);
}
}
database_column.Value = value;
database_row.Columns[column_def.Name] = database_column;
database_row.Columns[column_def.Name] = std::move(database_column);
}
assert(Id != -1); // no id found
database_row.RecordId = Id;
@@ -561,7 +562,7 @@ namespace Noggit
return std::optional<Structures::BlizzardDatabaseRow>();
}*/
Structures::BlizzardDatabaseRowDefinition ClientDatabaseTable::GetRecordDefinition() const
Structures::BlizzardDatabaseRowDefinition& ClientDatabaseTable::GetRecordDefinition() const
{
return Noggit::Project::CurrentProject::get()->ClientDatabase->TableRecordDefinition(_tableName);
}
@@ -594,9 +595,8 @@ namespace Noggit
if (query.next())
{
QSqlRecord record = query.record();
auto database_row = sqlRecordToDatabaseRow(record);
// QSqlRecord record = query.record(); // slow af
auto database_row = sqlRecordToDatabaseRow(query);
return database_row;
}
@@ -630,7 +630,7 @@ namespace Noggit
{
QString sql = QString("SELECT * FROM `%1`").arg(_table.getSqlTableName().c_str()); // ORDER BY ID ?
_query = ClientDatabase::executeQuery(sql);
_query = ClientDatabase::executeQuery(sql, true);
_querry_valid = _query.isActive(); // if query.exec ran properly
querryAdvance();
@@ -653,7 +653,6 @@ namespace Noggit
return _client_iterator.Next();
else
{
// if (_query.next())
if (!_querry_valid || !_hasNext)
{
assert(false);
@@ -661,12 +660,12 @@ namespace Noggit
}
// always store one row in advance to know if it's the last one
auto row = _table.sqlRecordToDatabaseRow(_nextRecord);
assert(row.RecordId != -1);
// auto row = _table.sqlRecordToDatabaseRow(_nextRecord);
assert(_nextRecord.RecordId != -1);
querryAdvance();
return row;
return _nextRecord;
}
}
@@ -675,7 +674,8 @@ namespace Noggit
{
if (_query.next())
{
_nextRecord = _query.record();
// _nextRecord = _query.record();
_nextRecord = _table.sqlRecordToDatabaseRow(_query);
_hasNext = true;
}
else
@@ -684,5 +684,4 @@ namespace Noggit
}
}
}

24
src/noggit/database/ClientDatabase.h
View File

@@ -45,12 +45,8 @@ namespace Noggit
{
friend class ClientDatabaseTable;
public:
// static ClientDatabase& instance()
// {
// static ClientDatabase _instance;
// return _instance;
// }
static void setDatabaseMode(DatabaseMode mode);
static DatabaseMode databaseMode(); // sql or client storage
static ClientDatabaseTable getTable(const std::string& tableName);
@@ -61,14 +57,10 @@ namespace Noggit
static void TODODeploySqlToClient();
static QSqlQuery executeQuery(const QString& sql);
static QSqlQuery executeQuery(const QString& sql, bool forward_only = true); // forward onl means can't browse query backward, but massively speeds up forward iteration
private:
// ClientDatabase() = default;
// ~ClientDatabase() = default;
// ClientDatabase(const ClientDatabase&) = delete;
// ClientDatabase& operator=(const ClientDatabase&) = delete;
static DatabaseMode _database_mode;
static Structures::BlizzardDatabaseRow clientRowById(const std::string& tableName, unsigned int id);
static Structures::BlizzardDatabaseRow sqlRowById(const std::string& tableName, unsigned int id);
@@ -88,7 +80,6 @@ namespace Noggit
private:
const ClientDatabaseTable& _table;
// DatabaseMode _mode;
// client
BlizzardDatabaseRecordCollection _client_iterator;
@@ -98,7 +89,8 @@ namespace Noggit
bool _querryHasStarted = false;
bool _querry_valid = false;
bool _hasNext = false;
QSqlRecord _nextRecord;
// QSqlRecord _nextRecord;
Structures::BlizzardDatabaseRow _nextRecord;
void querryAdvance();
};
@@ -111,6 +103,7 @@ namespace Noggit
private:
const std::string _tableName;
const QString _qtTableName;
const Structures::BlizzardDatabaseRowDefinition _row_definition;
public:
ClientDatabaseTable(std::string tableName);
@@ -120,7 +113,7 @@ namespace Noggit
unsigned int RecordCount() const;
int ColumnCount() const;
int getRecordSize() const;
Structures::BlizzardDatabaseRowDefinition GetRecordDefinition() const;
Structures::BlizzardDatabaseRowDefinition& GetRecordDefinition() const;
// get rows data
std::optional<Structures::BlizzardDatabaseRow> RecordById(unsigned int id) const;
@@ -148,7 +141,8 @@ namespace Noggit
std::vector<DbColumnFormat> recordFormat() const; // true record format for all columns, not array size/loc etc. eg returns all 17 columns for loc.
bool createSQLTableIfNotExist();
bool verifySqlTableIntegrity();
Structures::BlizzardDatabaseRow sqlRecordToDatabaseRow(const QSqlRecord& record) const;
// Structures::BlizzardDatabaseRow sqlRecordToDatabaseRow(const QSqlRecord& record) const;
Structures::BlizzardDatabaseRow sqlRecordToDatabaseRow(QSqlQuery& query) const;
};
}

44
src/noggit/database/clientDbDefinitions/MapDB.h
View File

@@ -0,0 +1,44 @@
#pragma once
#include <noggit/database/ClientDatabase.h>
// experimental
struct MapDbRow : BlizzardDatabaseLib::Structures::BlizzardDatabaseRow
{
uint AreaType() { return getUInt("InstanceType"); }; // uint
};
class MapDb : public Noggit::ClientDatabaseTable
{
public:
MapDb() :
ClientDatabaseTable("Map")
{}
uint MapID() ; // uint
std::optional<MapDbRow> mapRecordById(unsigned int id) { return static_cast<MapDbRow>(RecordById(id).value()); };
/// Fields
static const size_t MapID = 0; // uint
static const size_t InternalName = 1; // string
static const size_t AreaType = 2; // uint
static const size_t Flags = 3; // uint
static const size_t IsBattleground = 4; // uint
static const size_t Name = 5; // loc
static const size_t AreaTableID = 22; // uint
static const size_t MapDescriptionAlliance = 23; // loc
static const size_t MapDescriptionHorde = 40; // loc
static const size_t LoadingScreen = 57; // uint [LoadingScreen]
static const size_t minimapIconScale = 58; // uint [LoadingScreen]
static const size_t corpseMapID = 59; // iRefID Points to column 1, -1 if none
static const size_t corpseX = 60; // Float The X - Coord of the instance entrance
static const size_t corpseY = 61; // Float The Y - Coord of the instance entrance
static const size_t TimeOfDayOverride = 62; // Integer Set to - 1 for everything but Orgrimmar and Dalaran arena.For those, the time of day will change to this.
static const size_t ExpansionID = 63; // Integer Vanilla : 0, BC : 1, WotLK : 2
static const size_t RaidOffset = 64; // Integer
static const size_t NumberOfPlayers = 65; // Integer Used for reset time?
static std::string getMapName(int pMapID);
static int findMapName(const std::string& map_name);
};

10
src/noggit/ui/tools/MapCreationWizard/Ui/MapCreationWizard.cpp
View File

@@ -93,6 +93,16 @@ MapCreationWizard::MapCreationWizard(std::shared_ptr<Project::NoggitProject> pro
auto testtable = ClientDatabase::getTable("WMOAreaTable");
// auto& testtable = Noggit::Project::CurrentProject::get()->ClientDatabase->LoadTable("WMOAreaTable", readFileAsIMemStream);
//
// for (DBCFile::Iterator i = gWMOAreaTableDB.begin(); i != gWMOAreaTableDB.end(); ++i)
// {
// unsigned int lol = i->getUInt(WMOAreaTableDB::ID);
// for (int ii = 0; ii < WMOAreaTableDB::Name; ii++)
// {
// unsigned int lol = i->getUInt(ii);
// }
// i->getLocalizedString(WMOAreaTableDB::Name);
// }
qint64 elapsedMs = timer.elapsed();
Log << "gettable() in : " << elapsedMs << "ms" << std::endl;

6
src/noggit/ui/windows/noggitWindow/NoggitWindow.cpp
View File

@@ -78,7 +78,8 @@ namespace Noggit::Ui::Windows
}
else
{
LogError << "NoggitWindow() : Unsupported project version, skipping loading DBCs." << std::endl;
assert(false); // TODO
LogError << "NoggitWindow() : Unsupported project version, skipping loading DBCs." << std::endl;
}
_settings = new settings(this);
@@ -86,6 +87,9 @@ namespace Noggit::Ui::Windows
QSettings settings;
// connect to databases
bool use_mysql = settings.value("project/mysql/enabled", false).toBool();
ClientDatabase::setDatabaseMode(use_mysql ? DatabaseMode::Sql : DatabaseMode::ClientStorage);
if (use_mysql)
{
auto host = settings.value("project/mysql/server").toString();