From 578c83a72c41dcb25b5e12c6ff9741b5429003c8 Mon Sep 17 00:00:00 2001 From: T1ti <40864460+T1ti@users.noreply.github.com> Date: Tue, 23 Sep 2025 01:31:41 +0200 Subject: [PATCH] rewrite ClientDatabase and move to new sql api --- src/external/blizzard-database-library | 2 +- src/noggit/database/ClientDatabase.cpp | 610 +++++++++++------- src/noggit/database/ClientDatabase.h | 150 +++-- .../Ui/MapCreationWizard.cpp | 71 +- .../ui/tools/PresetEditor/Ui/PresetEditor.cpp | 5 +- .../components/BuildMapListComponent.cpp | 5 +- 6 files changed, 529 insertions(+), 314 deletions(-) diff --git a/src/external/blizzard-database-library b/src/external/blizzard-database-library index 260564ee..33bab9f0 160000 --- a/src/external/blizzard-database-library +++ b/src/external/blizzard-database-library @@ -1 +1 @@ -Subproject commit 260564ee1f379b393959477867855ff35f3b63d6 +Subproject commit 33bab9f08cdcb183b030dd0b24f9d0a47d5b1d01 diff --git a/src/noggit/database/ClientDatabase.cpp b/src/noggit/database/ClientDatabase.cpp index 459d471b..d919cb03 100644 --- a/src/noggit/database/ClientDatabase.cpp +++ b/src/noggit/database/ClientDatabase.cpp @@ -7,6 +7,7 @@ #include #include #include +#include namespace Noggit { @@ -18,94 +19,40 @@ namespace Noggit }; - std::optional ClientDatabase::getRowById(const std::string& tableName, unsigned int id) + DatabaseMode ClientDatabase::databaseMode() + { + bool setting_use_sql_db = true; // todo QSETTING + QSettings settings; + setting_use_sql_db = settings.value("project/mysql/enabled", false).toBool(); + + return setting_use_sql_db ? DatabaseMode::Sql : DatabaseMode::ClientStorage; + } + + ClientDatabaseTable ClientDatabase::getTable(const std::string& tableName) + { + return ClientDatabaseTable(tableName); + } + + bool ClientDatabaseTable::UploadDBCtoDB() { - bool setting_use_sql_db = false; // todo QSETTING - auto row = Structures::BlizzardDatabaseRow(-1); + auto sql_table_name = getSqlTableName(); - if (setting_use_sql_db) - row = sqlRowById(tableName, id); - else - row = clientRowById(tableName, id); - - if (row.RecordId == -1) - return std::nullopt; - else - return row; - } - - bool ClientDatabase::testUploadDBCtoDB(const BlizzardDatabaseLib::BlizzardDatabaseTable& table) - { - auto& db_mgr = Noggit::Sql::SqlDatabaseManager::instance(); - bool valid_conn = db_mgr.testConnection(Noggit::Sql::SQLDbType::Noggit); - if (!valid_conn) + /* + if (!verifySqlTableIntegrity()) + { + Log << "Table " << sql_table_name << "does not exist or has wrong structure."; + qDebug() << "Table " << sql_table_name.c_str() << "does not exist or has wrong structure."; return false; - - auto table_name = table.Name(); - // Noggit::Project::CurrentProject::get()->projectVersion; // expension, not exact build id - unsigned int build_id = Noggit::Project::CurrentProject::get()->buildId(); - - // check if table exists - QString sql_table_name = getSqlTableName(table_name, build_id).c_str(); - - auto noggit_db = db_mgr.noggitDatabase(); - - // table integrity check - bool table_is_valid = true; - bool fresh_table = false; - QSqlRecord sql_rec = noggit_db.record(sql_table_name); - - // noggit_db.tables().contains(sql_table_name) is bugged with current qt version and mysql 8 - QSqlQuery query_show(noggit_db); - if (!query_show.exec("SHOW TABLES")) - { - qWarning() << "Failed to list tables:" << query_show.lastError().text(); - return false; - } - QStringList tables; - while (query_show.next()) - { - tables << query_show.value(0).toString(); - } - - if (tables.contains(sql_table_name)) - { - // this is also bugged... - // if (sql_rec.isEmpty()) - // { - // table_is_valid = false; - // } - // else - // { - // // TODO verify db structure, just column count for now - // if (table.ColumnCount() != sql_rec.count()) - // { - // assert(false); - // table_is_valid = false; - // } - // } - } - else // table doesn't exist - { - // create table - table_is_valid = createSQLTableIfNotExist(table); - fresh_table = true; - } - - if (!table_is_valid) - { - Log << "Table " << sql_table_name.toStdString() << "does not exist or has wrong structure."; - qDebug() << "Table " << sql_table_name << "does not exist or has wrong structure."; - return false; - } + }*/ + qDebug() << "Populating empty Table " << sql_table_name.c_str(); // insert if fresh_table, otherwise replace? - auto row_definition = table.GetRecordDefinition(); - auto sql_record_format = recordFormat(table_name); + auto row_definition = GetRecordDefinition(); + auto sql_record_format = recordFormat(); - auto client_table_iterator = table.Records(); + auto client_table_iterator = getClientTable().Records(); // empty table, nothing to insert if (!client_table_iterator.HasRecords()) @@ -118,6 +65,9 @@ namespace Noggit } int colCount = column_names.size(); + auto& db_mgr = Noggit::Sql::SqlDatabaseManager::instance(); + auto noggit_db = db_mgr.noggitDatabase(); + QSqlQuery query(noggit_db); // Start bulk insert /////////////////////////////// QElapsedTimer timer; @@ -126,8 +76,6 @@ namespace Noggit const int batchSize = 2000; int rowCount = 0; - QSqlQuery query(noggit_db); - noggit_db.transaction(); // query.exec("SET UNIQUE_CHECKS=0;"); @@ -184,7 +132,7 @@ namespace Noggit QString sql; sql.reserve(min_size); sql = QString("INSERT INTO `%1` (%2) VALUES ") - .arg(sql_table_name) + .arg(sql_table_name.c_str()) .arg(column_names.join(", ")); sql += rowBuffer.join(","); @@ -207,7 +155,7 @@ namespace Noggit QString sql; sql.reserve(min_size); sql = QString("INSERT INTO `%1` (%2) VALUES ") - .arg(sql_table_name) + .arg(sql_table_name.c_str()) .arg(column_names.join(", ")); sql += rowBuffer.join(","); @@ -225,146 +173,52 @@ namespace Noggit // benchmark qint64 elapsedMs = timer.elapsed(); - qDebug() << "Inserted" << table.RecordCount() << "rows in" - << elapsedMs << "ms (" - << (table.RecordCount() * 1000.0 / elapsedMs) << " rows/sec)"; + qDebug() << "Inserted" << getClientTable().RecordCount() << "rows in" << elapsedMs << "ms (" + << (getClientTable().RecordCount() * 1000.0 / elapsedMs) << " rows/sec)"; - Log << "Inserted " << table.RecordCount() << " rows in " - << elapsedMs << "ms (" - << (table.RecordCount() * 1000.0 / elapsedMs) << " rows/sec)" << std::endl; + Log << "Inserted " << getClientTable().RecordCount() << " rows in " << elapsedMs << "ms (" + << (getClientTable().RecordCount() * 1000.0 / elapsedMs) << " rows/sec)" << std::endl; return true; } - // get from local dbc data memory stream in BlizzardDatabaseLib::BlizzardDatabase - Structures::BlizzardDatabaseRow ClientDatabase::clientRowById(const std::string& tableName, unsigned int id) + // executes query in client db and checks errors + // use isActive to check if it properly ran, not isValid. + QSqlQuery ClientDatabase::executeQuery(const QString& sql) { - auto& table = Noggit::Project::CurrentProject::get()->ClientDatabase->LoadTable(tableName, readFileAsIMemStream); - auto record = table.RecordById(id); + auto db_mgr = Noggit::Sql::SqlDatabaseManager::instance().noggitDatabase(); + QSqlQuery query(Noggit::Sql::SqlDatabaseManager::instance().noggitDatabase()); - return record; + qDebug() << "Executing query : " << sql; + QElapsedTimer timer; + timer.start(); + if (!query.exec(sql)) + { + LogError << "SQL query failed:" << query.lastError().text().toStdString(); + LogError << "Query:" << sql.toStdString(); + // throw SqlException("Query failed: " + query.lastError().text() + "\nQuery: " + sql); + assert(false); + return QSqlQuery(); // invalid query + } + + qint64 elapsedMs = timer.elapsed(); + qDebug() << "Executed query in " << elapsedMs << "ms"; + + return query; } - // get from SQL request to noggit db - // never use this function for more than 1 rows, implement a new bulk function - Structures::BlizzardDatabaseRow ClientDatabase::sqlRowById(const std::string& tableName, unsigned int id) + bool ClientDatabaseTable::createSQLTableIfNotExist() { - auto& db_mgr = Noggit::Sql::SqlDatabaseManager::instance(); - // Test connection ? + auto row_definition = GetRecordDefinition(); - auto noggit_db = db_mgr.noggitDatabase(); + const std::string sql_table_name = getSqlTableName(); - auto row_definition = Noggit::Project::CurrentProject::get()->ClientDatabase->TableRecordDefinition(tableName); + auto db_record_format = recordFormat(); + assert(db_record_format.size() == getClientTable().ColumnCount()); - QString sql_table_name = getSqlTableName(tableName).c_str(); - QSqlQuery query(noggit_db); - QString sql = QString("SELECT * FROM %1 WHERE ID = :id").arg(sql_table_name); - - query.prepare(sql); - query.bindValue(":id", id); - - if (!query.exec()) - { - qWarning() << "Query exec failed:" << query.lastError().text(); - return BlizzardDatabaseLib::Structures::BlizzardDatabaseRow(); - } - - if (query.next()) - { - QSqlRecord record = query.record(); - - auto database_row = Structures::BlizzardDatabaseRow(id); - - // test db def///////////////////////// - { - auto record_db_def = recordFormat(tableName); - if (record.count() != record_db_def.size()) - { - // error : definition deosn't match db structure - assert(false); - return Structures::BlizzardDatabaseRow(-1); - } - - // Tests construct row from query - // TODOOOOOOOOOOOOOOO - - for (int i = 0; i < record_db_def.size(); ++i) - { - auto& column_db_def = record_db_def[i]; - QSqlField db_field = record.field(i); - - assert(column_db_def.Name == record.fieldName(i).toStdString()); - // assert(column_db_def.Type == db_field.type()); - } - }///////////////////////////////////////// - - int field_idx = 0; - for (int column_def_idx = 0; column_def_idx < row_definition.ColumnDefinitions.size(); ++column_def_idx) - { - auto& column_def = row_definition.ColumnDefinitions[column_def_idx]; - auto database_column = Structures::BlizzardDatabaseColumn(); - - auto value = std::string(); - if (column_def.Type == "locstring") - { - std::vector localizedValues = std::vector(); - for (int loc_idx = 0; loc_idx < 16; loc_idx++) - { - localizedValues.push_back(query.value(field_idx++).toString().toStdString()); - } - - database_column.Values = localizedValues; - database_row.Columns[column_def.Name] = database_column; - - // currently loc mask is set to a separate column because wdbc reader does it. - auto loc_mask_column = Structures::BlizzardDatabaseColumn(); - loc_mask_column.Value = query.value(field_idx++).toString().toStdString(); - database_row.Columns[column_def.Name + "_flags"] = loc_mask_column; - } - else // every other type than locstring - { - if (column_def.arrLength > 1) // array - { - for (int i = 0; i < column_def.arrLength; i++) - { - auto Value = query.value(field_idx++); - database_column.Values.push_back(Value.toString().toStdString()); - } - } - else // single value - { - auto Value = query.value(field_idx++); - value = Value.toString().toStdString(); - } - } - database_column.Value = value; - database_row.Columns[column_def.Name] = database_column; - } - - return database_row; - } - else - { - qWarning() << "No row found in" << tableName.c_str() << "for ID =" << id; - return BlizzardDatabaseLib::Structures::BlizzardDatabaseRow(); - } - - } - - bool ClientDatabase::createSQLTableIfNotExist(const BlizzardDatabaseLib::BlizzardDatabaseTable& table) - { - const std::string table_name = table.Name(); - auto row_definition = table.GetRecordDefinition(); - - const std::string sql_table_name = getSqlTableName(table_name); std::string statement = std::format("CREATE TABLE IF NOT EXISTS `{}` (", sql_table_name); std::string primary_key_name; - - auto db_record_format = recordFormat(table_name); - - assert(db_record_format.size() == table.ColumnCount()); - for (auto& db_column_format : db_record_format) { statement += std::format("`{}` {}", db_column_format.Name, db_column_format.Type); @@ -422,32 +276,19 @@ namespace Noggit } else { - qDebug() << "Table " << table_name.c_str() << " created."; + qDebug() << "Table " << sql_table_name.c_str() << " created."; + + UploadDBCtoDB(); } return success; } - std::string ClientDatabase::getSqlTableName(const std::string& db_name, unsigned int build_id) - { - if (build_id == 0) - build_id = Noggit::Project::CurrentProject::get()->buildId(); - - std::string table = std::format("db_{}_{}", db_name, build_id); - - // convert to lowercase for compatibility with SQL - std::transform(table.begin(), table.end(), table.begin(), - [](unsigned char c) { return std::tolower(c); }); - - return table; - } - - std::vector ClientDatabase::recordFormat(const std::string& table_name) + std::vector ClientDatabaseTable::recordFormat() const { auto record_format = std::vector(); - auto row_definition = Noggit::Project::CurrentProject::get()->ClientDatabase->TableRecordDefinition(table_name); - + auto row_definition = GetRecordDefinition(); for (int col_idx = 0; col_idx < row_definition.ColumnDefinitions.size(); col_idx++) { auto& column_def = row_definition.ColumnDefinitions[col_idx]; @@ -527,4 +368,321 @@ namespace Noggit return record_format; } + bool ClientDatabaseTable::verifySqlTableIntegrity() + { + auto& db_mgr = Noggit::Sql::SqlDatabaseManager::instance(); + bool valid_conn = db_mgr.testConnection(Noggit::Sql::SQLDbType::Noggit); + if (!valid_conn) + return false; + + // check if table exists + QString sql_table_name = getSqlTableName().c_str(); + + auto noggit_db = db_mgr.noggitDatabase(); + + // table integrity check + bool table_is_valid = true; + bool fresh_table = false; + + // noggit_db.tables().contains(sql_table_name) is bugged with current qt version and mysql 8 + QSqlQuery query_show(noggit_db); + if (!query_show.exec("SHOW TABLES")) + { + qWarning() << "Failed to list tables:" << query_show.lastError().text(); + return false; + } + QStringList tables; + while (query_show.next()) + { + tables << query_show.value(0).toString(); + } + + if (tables.contains(sql_table_name)) + { + // this is also bugged... + // QSqlRecord sql_rec = noggit_db.record(sql_table_name); + // if (sql_rec.isEmpty()) + // { + // table_is_valid = false; + // } + // else + // { + // // TODO verify db structure, just column count for now + // if (table.ColumnCount() != sql_rec.count()) + // { + // assert(false); + // table_is_valid = false; + // } + // } + } + else // table doesn't exist + { + // create table + table_is_valid = createSQLTableIfNotExist(); + fresh_table = true; + } + + + return table_is_valid; + } + + Structures::BlizzardDatabaseRow ClientDatabaseTable::sqlRecordToDatabaseRow(const QSqlRecord& record) const + { + auto row_definition = GetRecordDefinition(); + + auto database_row = Structures::BlizzardDatabaseRow(-1); + + int Id = -1; + int field_idx = 0; + for (int column_def_idx = 0; column_def_idx < row_definition.ColumnDefinitions.size(); ++column_def_idx) + { + auto& column_def = row_definition.ColumnDefinitions[column_def_idx]; + auto database_column = Structures::BlizzardDatabaseColumn(); + + auto value = std::string(); + if (column_def.Type == "locstring") + { + std::vector localizedValues = std::vector(); + for (int loc_idx = 0; loc_idx < 16; loc_idx++) + { + localizedValues.push_back(record.value(field_idx++).toString().toStdString()); + } + + database_column.Values = localizedValues; + database_row.Columns[column_def.Name] = database_column; + + // currently loc mask is set to a separate column because wdbc reader does it. + auto loc_mask_column = Structures::BlizzardDatabaseColumn(); + loc_mask_column.Value = record.value(field_idx++).toString().toStdString(); + database_row.Columns[column_def.Name + "_flags"] = loc_mask_column; + } + else // every other type than locstring + { + if (column_def.arrLength > 1) // array + { + for (int i = 0; i < column_def.arrLength; i++) + { + database_column.Values.push_back(record.value(field_idx++).toString().toStdString()); + } + } + else // single value + { + value = record.value(field_idx++).toString().toStdString(); + + if (column_def.isID) + Id = std::stoi(value); + } + } + database_column.Value = value; + database_row.Columns[column_def.Name] = database_column; + } + assert(Id != -1); // no id found + database_row.RecordId = Id; + + return database_row; + } + + ClientDatabaseTable::ClientDatabaseTable(std::string tableName) + : _tableName(tableName), _qtTableName(tableName.c_str()) + { + if (ClientDatabase::databaseMode() == DatabaseMode::Sql) + verifySqlTableIntegrity(); // verifySqlTableIntegrity()->createtableifnotexists()->UploadDBCtoDB() + }; + + unsigned int ClientDatabaseTable::RecordCount() const + { + unsigned int client_count = getClientTable().RecordCount(); + + if (ClientDatabase::databaseMode() == DatabaseMode::Sql) + { + QString sql = QString("SELECT COUNT(*) FROM `%1`").arg(getSqlTableName().c_str()); + QSqlQuery query = ClientDatabase::executeQuery(sql); + + if (query.isActive()) + { + if (query.next()) + assert(query.value(0).toUInt() == client_count); + } + } + return client_count; + } + + int ClientDatabaseTable::ColumnCount() const + { + // get from parsed definition + int def_column_count = recordFormat().size(); + + int client_count = getClientTable().ColumnCount(); + assert(def_column_count == client_count); + + if (ClientDatabase::databaseMode() == DatabaseMode::Sql) + { + auto db = Noggit::Sql::SqlDatabaseManager::instance().noggitDatabase(); + QSqlRecord rec = db.record(QString::fromStdString(getSqlTableName())); + int db_count = rec.count(); + assert(db_count == def_column_count); + } + + return def_column_count; + } + + std::optional ClientDatabaseTable::RecordById(unsigned int id) const + { + auto row = Structures::BlizzardDatabaseRow(-1); + + if (ClientDatabase::databaseMode() == DatabaseMode::Sql) + row = sqlRowById(id); + else + row = clientRowById(id); + + if (row.RecordId == -1) + return std::nullopt; + else + return row; + } + + Noggit::DatabaseRecordCollection ClientDatabaseTable::Records() const + { + return Noggit::DatabaseRecordCollection(*this); + }; + + /* + std::optional ClientDatabaseTable::RecordByPosition(unsigned int positionId) const + { + // TODO + auto row = Structures::BlizzardDatabaseRow(-1); + if (ClientDatabase::databaseMode() == DatabaseMode::Sql) + { + // We shouldn't do this with SQL table + } + else + row = getClientTable().RecordByPosition(positionId); + + return std::optional(); + }*/ + + Structures::BlizzardDatabaseRowDefinition ClientDatabaseTable::GetRecordDefinition() const + { + return Noggit::Project::CurrentProject::get()->ClientDatabase->TableRecordDefinition(_tableName); + } + + BlizzardDatabaseLib::BlizzardDatabaseTable& ClientDatabaseTable::getClientTable() const + { + return Noggit::Project::CurrentProject::get()->ClientDatabase->LoadTable(_tableName, readFileAsIMemStream); + } + + // get from local dbc data memory stream in BlizzardDatabaseLib::BlizzardDatabase + Structures::BlizzardDatabaseRow ClientDatabaseTable::clientRowById(unsigned int id) const + { + auto record = getClientTable().RecordById(id); + return record; + } + + // get from SQL request to noggit db + // never use this function for more than 1 rows, implement a new bulk function + Structures::BlizzardDatabaseRow ClientDatabaseTable::sqlRowById(unsigned int id) const + { + QString sql_table_name = getSqlTableName().c_str(); + QString sql = QString("SELECT * FROM `%1` WHERE ID = %2").arg(sql_table_name).arg(id); + + auto query = ClientDatabase::executeQuery(sql); + + if (!query.isActive()) + return BlizzardDatabaseLib::Structures::BlizzardDatabaseRow(-1); + + // auto row_definition = GetRecordDefinition(); + + if (query.next()) + { + QSqlRecord record = query.record(); + + auto database_row = sqlRecordToDatabaseRow(record); + + return database_row; + } + else + { + LogError << "SQL : No row found in" << sql_table_name.toStdString() << "for ID =" << id; + qWarning() << "SQL : No row found in" << sql_table_name << "for ID =" << id; + return BlizzardDatabaseLib::Structures::BlizzardDatabaseRow(); + } + + } + + const std::string ClientDatabaseTable::getSqlTableName(unsigned int build_id) const + { + if (build_id == 0) + build_id = Noggit::Project::CurrentProject::get()->buildId(); + + std::string table = std::format("db_{}_{}", _tableName, build_id); + + // convert to lowercase for compatibility with SQL + std::transform(table.begin(), table.end(), table.begin(), + [](unsigned char c) { return std::tolower(c); }); + + return table; + } + + DatabaseRecordCollection::DatabaseRecordCollection(const ClientDatabaseTable& table) + :_table(table), /*_mode(mode),*/ _client_iterator(_table.getClientTable().Records()) + { + if (ClientDatabase::databaseMode() == DatabaseMode::Sql) + { + QString sql = QString("SELECT * FROM `%1`").arg(_table.getSqlTableName().c_str()); // ORDER BY ID ? + + _query = ClientDatabase::executeQuery(sql); + _querry_valid = _query.isActive(); // if query.exec ran properly + + querryAdvance(); + } + } + + bool DatabaseRecordCollection::HasRecords() + { + if (ClientDatabase::databaseMode() == DatabaseMode::ClientStorage) + return _client_iterator.HasRecords(); + else + { + return _querry_valid && _hasNext; + } + } + + Structures::BlizzardDatabaseRow DatabaseRecordCollection::Next() + { + if (ClientDatabase::databaseMode() == DatabaseMode::ClientStorage) + return _client_iterator.Next(); + else + { + // if (_query.next()) + if (!_querry_valid || !_hasNext) + { + assert(false); + return Structures::BlizzardDatabaseRow(); // empty + } + + // always store one row in advance to know if it's the last one + auto row = _table.sqlRecordToDatabaseRow(_nextRecord); + assert(row.RecordId != -1); + + querryAdvance(); + + return row; + } + + } + + void DatabaseRecordCollection::querryAdvance() + { + if (_query.next()) + { + _nextRecord = _query.record(); + _hasNext = true; + } + else + { + _hasNext = false; + } + } + + } \ No newline at end of file diff --git a/src/noggit/database/ClientDatabase.h b/src/noggit/database/ClientDatabase.h index bd94573a..de8bc614 100644 --- a/src/noggit/database/ClientDatabase.h +++ b/src/noggit/database/ClientDatabase.h @@ -6,15 +6,15 @@ #include +#include +#include + constexpr const char* dbc_string_loc_names[16] = { "enUS", "koKR", "frFR", "deDE", "zhCN", "zhTW", "esES", "esMX", "ruRU", "jaJP", "ptPT", "itIT", "unk_12", "unk_13", "unk_14", "unk_15" }; using namespace BlizzardDatabaseLib; - - - namespace Noggit { struct DbColumnFormat @@ -27,64 +27,41 @@ namespace Noggit bool isSigned = true; }; - // interface table that gets data either from sql or raw dbc - class ClientDatabaseTable + enum class DatabaseMode { - private: - const std::string _tableName; - - public: - unsigned int RecordCount() const; - - // column count from file header, not definition file - int ColumnCount() const - { - return static_cast(_tableReader->FieldCount()); - } - - std::string Name() const - { - return _tableName; - } - - Structures::BlizzardDatabaseRow RecordById(unsigned int id) const - { - return _tableReader->RecordById(id); - } - - Structures::BlizzardDatabaseRow RecordByPosition(unsigned int positionId) const - { - return _tableReader->Record(positionId); - } - - BlizzardDatabaseRecordCollection Records() const - { - return BlizzardDatabaseRecordCollection(_tableReader); - } - - Structures::BlizzardDatabaseRowDefinition GetRecordDefinition() const - { - return _tableReader->RecordDefinition(); - } - - + Sql, + ClientStorage }; + struct SqlException : public std::runtime_error + { + SqlException(const QString& msg) + : std::runtime_error(msg.toStdString()) {} + }; + + class Noggit::ClientDatabaseTable; // calls client or server db adaptively. so /sql/ is not really a good location class ClientDatabase { + friend class ClientDatabaseTable; public: // static ClientDatabase& instance() // { // static ClientDatabase _instance; // return _instance; // } + + static DatabaseMode databaseMode(); // sql or client storage + + static ClientDatabaseTable getTable(const std::string& tableName); + + static void saveTable(const std::string& tableName); - static std::optional getRowById(const std::string& tableName, unsigned int id); // constructs a row either from db or client - - static bool testUploadDBCtoDB(const BlizzardDatabaseLib::BlizzardDatabaseTable& table); + // static std::optional getRowById(const std::string& tableName, unsigned int id); // constructs a row either from db or client static void TODODeploySqlToClient(); + + static QSqlQuery executeQuery(const QString& sql); private: // ClientDatabase() = default; @@ -96,13 +73,82 @@ namespace Noggit static Structures::BlizzardDatabaseRow clientRowById(const std::string& tableName, unsigned int id); static Structures::BlizzardDatabaseRow sqlRowById(const std::string& tableName, unsigned int id); - static bool createSQLTableIfNotExist(const BlizzardDatabaseLib::BlizzardDatabaseTable& table); - - static std::string getSqlTableName(const std::string& db_name, unsigned int build_id = 0); // get automatically from project if default(0) - - static std::vector recordFormat(const std::string& table_name); // true record format for all columns, not array size/loc etc. eg returns all 17 columns for loc. - + }; + // TODO can subclass BlizzardDatabaseRecordCollection instead + class DatabaseRecordCollection + { + public: + DatabaseRecordCollection(const ClientDatabaseTable& table); + bool HasRecords(); + Structures::BlizzardDatabaseRow Next(); + // Structures::BlizzardDatabaseRow First(); + // Structures::BlizzardDatabaseRow Last(); + + private: + const ClientDatabaseTable& _table; + // DatabaseMode _mode; + + // client + BlizzardDatabaseRecordCollection _client_iterator; + + // sql stuff + QSqlQuery _query; + bool _querryHasStarted = false; + bool _querry_valid = false; + bool _hasNext = false; + QSqlRecord _nextRecord; + void querryAdvance(); + }; + + // interface table that gets data either from sql or raw dbc + // TODO : can just make BlizzardDatabaseTable subclass this. + class ClientDatabaseTable + { + friend class DatabaseRecordCollection; + + private: + const std::string _tableName; + const QString _qtTableName; + + public: + ClientDatabaseTable(std::string tableName); + + // table info + const std::string Name() const { return _tableName; }; + unsigned int RecordCount() const; + int ColumnCount() const; + int getRecordSize() const; + Structures::BlizzardDatabaseRowDefinition GetRecordDefinition() const; + + // get rows data + std::optional RecordById(unsigned int id) const; + // std::optional RecordByPosition(unsigned int positionId) const; + // CheckIfIdExists + Noggit::DatabaseRecordCollection Records() const;// { return Noggit::DatabaseRecordCollection(*this); }; + + // modify data + // Record addRecord(size_t id, size_t id_field = 0); + // Record addRecordCopy(size_t id, size_t id_from, size_t id_field = 0); + // void removeRecord(size_t id, size_t id_field = 0); + // int getEmptyRecordID(size_t id_field = 0); + + + private: + // for internal use only, get the client storage + BlizzardDatabaseLib::BlizzardDatabaseTable& getClientTable() const; + + Structures::BlizzardDatabaseRow clientRowById(unsigned int id) const; + Structures::BlizzardDatabaseRow sqlRowById(unsigned int id) const; + + // sql helpers + bool UploadDBCtoDB(); + const std::string getSqlTableName(unsigned int build_id = 0) const; // get automatically from project if default(0) + std::vector recordFormat() const; // true record format for all columns, not array size/loc etc. eg returns all 17 columns for loc. + bool createSQLTableIfNotExist(); + bool verifySqlTableIntegrity(); + Structures::BlizzardDatabaseRow sqlRecordToDatabaseRow(const QSqlRecord& record) const; + }; } diff --git a/src/noggit/ui/tools/MapCreationWizard/Ui/MapCreationWizard.cpp b/src/noggit/ui/tools/MapCreationWizard/Ui/MapCreationWizard.cpp index 30f728bc..154559ca 100755 --- a/src/noggit/ui/tools/MapCreationWizard/Ui/MapCreationWizard.cpp +++ b/src/noggit/ui/tools/MapCreationWizard/Ui/MapCreationWizard.cpp @@ -35,6 +35,7 @@ #include #include #include +#include #include @@ -82,10 +83,37 @@ MapCreationWizard::MapCreationWizard(std::shared_ptr pro _corpse_map_id->addItem("None"); _corpse_map_id->setItemData(0, QVariant (-1)); - // Fill selector combo + // TEST BENCHMARK SQL STUFF///////////////////////// + /* + { + Log << "Iterating table : " << "WMOAreaTable" << std::endl; + QElapsedTimer timer; + timer.start(); + + auto testtable = ClientDatabase::getTable("WMOAreaTable"); + // auto& testtable = Noggit::Project::CurrentProject::get()->ClientDatabase->LoadTable("WMOAreaTable", readFileAsIMemStream); + + qint64 elapsedMs = timer.elapsed(); + Log << "gettable() in : " << elapsedMs << "ms" << std::endl; + + auto iterator = testtable.Records(); + while (iterator.HasRecords()) + { + auto record = iterator.Next(); + } + elapsedMs = timer.elapsed(); + Log << "fully iterated table in : " << elapsedMs << "ms" << std::endl; + + }*/ + ///////////////////////////////////////////////////// + + + // Fill selector combo const auto& table = std::string("Map"); - auto& mapTable = _project->ClientDatabase->LoadTable(table, readFileAsIMemStream); + + auto mapTable = ClientDatabase::getTable(table); + // auto& mapTable = Noggit::Project::CurrentProject::get()->ClientDatabase->LoadTable(table, readFileAsIMemStream); int count = 0; auto iterator = mapTable.Records(); @@ -106,8 +134,6 @@ MapCreationWizard::MapCreationWizard(std::shared_ptr pro count++; } - _project->ClientDatabase->UnloadTable("Map"); - auto add_btn = new QPushButton("New",this); add_btn->setIcon(Noggit::Ui::FontAwesomeIcon(Noggit::Ui::FontAwesome::plus)); layout_selector->addWidget(add_btn); @@ -630,24 +656,11 @@ void MapCreationWizard::selectMap(int map_id) // int map_id = world->getMapID(); - auto& table = _project->ClientDatabase->LoadTable("Map", readFileAsIMemStream); - auto record = table.RecordById(map_id); - - - /// test area, delete later ///////////// - QSettings settings; - bool use_mysql = settings.value("project/mysql/enabled", false).toBool(); - if (use_mysql) - { - // bool valid_conn = mysql::testConnection(true); - auto& test_table = _project->ClientDatabase->LoadTable("ItemDisplayInfo", readFileAsIMemStream); - Noggit::ClientDatabase::testUploadDBCtoDB(test_table); - - // TODO : crashes if not unloading - //_project->ClientDatabase->UnloadTable("AreaTable"); - - } - /////////////////////////////// + auto table = ClientDatabase::getTable("Map"); + auto rec_opt = table.RecordById(map_id); + if (!rec_opt) + return; + auto& record = *rec_opt; _cur_map_id = map_id; @@ -740,9 +753,7 @@ void MapCreationWizard::selectMap(int map_id) _max_players->setValue(std::atoi(maxPlayers.c_str())); - _project->ClientDatabase->UnloadTable("Map"); - - auto& difficulty_table = _project->ClientDatabase->LoadTable("MapDifficulty", readFileAsIMemStream); + auto difficulty_table = ClientDatabase::getTable("MapDifficulty"); auto iterator = difficulty_table.Records(); @@ -763,7 +774,6 @@ void MapCreationWizard::selectMap(int map_id) _difficulty_type->insertItem(difficulty_type, diff_text.c_str(), QVariant(record_id)); } } - _project->ClientDatabase->UnloadTable("MapDifficulty"); _difficulty_type->setCurrentIndex(0); selectMapDifficulty(); @@ -779,8 +789,11 @@ void MapCreationWizard::selectMapDifficulty() if (!selected_difficulty_id) return; - auto& difficulty_table = _project->ClientDatabase->LoadTable("MapDifficulty", readFileAsIMemStream); - auto record = difficulty_table.RecordById(selected_difficulty_id); + auto difficulty_table = ClientDatabase::getTable("MapDifficulty"); + auto rec_opt = difficulty_table.RecordById(selected_difficulty_id); + if (!rec_opt) + return; + auto& record = *rec_opt; //_difficulty_type; _difficulty_req_message->fill(record, "Message_lang"); @@ -790,8 +803,6 @@ void MapCreationWizard::selectMapDifficulty() _difficulty_max_players->setValue(std::atoi(record.Columns["MaxPlayers"].Value.c_str())); _difficulty_string->setText(record.Columns["Difficultystring"].Value.c_str()); - - _project->ClientDatabase->UnloadTable("MapDifficulty"); } void MapCreationWizard::wheelEvent(QWheelEvent* event) diff --git a/src/noggit/ui/tools/PresetEditor/Ui/PresetEditor.cpp b/src/noggit/ui/tools/PresetEditor/Ui/PresetEditor.cpp index 73a35b60..34d380c7 100755 --- a/src/noggit/ui/tools/PresetEditor/Ui/PresetEditor.cpp +++ b/src/noggit/ui/tools/PresetEditor/Ui/PresetEditor.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include @@ -85,7 +86,8 @@ PresetEditorWidget::PresetEditorWidget(std::shared_ptr p ui->worldSelector->setItemData(0, QVariant(-1)); const auto& table = std::string("Map"); - auto& mapTable = _project->ClientDatabase->LoadTable(table, readFileAsIMemStream); + + auto mapTable = ClientDatabase::getTable(table); int count = 1; auto iterator = mapTable.Records(); @@ -108,7 +110,6 @@ PresetEditorWidget::PresetEditorWidget(std::shared_ptr p count++; } - _project->ClientDatabase->UnloadTable("Map"); // Handle minimap widget diff --git a/src/noggit/ui/windows/noggitWindow/components/BuildMapListComponent.cpp b/src/noggit/ui/windows/noggitWindow/components/BuildMapListComponent.cpp index 4660f38c..26623542 100755 --- a/src/noggit/ui/windows/noggitWindow/components/BuildMapListComponent.cpp +++ b/src/noggit/ui/windows/noggitWindow/components/BuildMapListComponent.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include @@ -25,7 +26,7 @@ void BuildMapListComponent::buildMapList(Noggit::Ui::Windows::NoggitWindow* pare } const auto& table = std::string("Map"); - auto& map_table = parent->_project->ClientDatabase->LoadTable(table, readFileAsIMemStream); + auto map_table = ClientDatabase::getTable(table); auto iterator = map_table.Records(); auto pinned_maps = std::vector(); @@ -126,6 +127,4 @@ void BuildMapListComponent::buildMapList(Noggit::Ui::Windows::NoggitWindow* pare item->setData(Qt::UserRole, QVariant(map.map_id)); parent->_continents_table->setItemWidget(item, map_list_item); } - - parent->_project->ClientDatabase->UnloadTable(table); }