natsirt/noggit-red
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix dbc shared reader bug when not unloading, and optimize bulk query

Browse Source
This commit is contained in:
T1ti
2025-09-19 17:35:43 +02:00
parent 77bc08b70b
commit c5b1ca786c
6 changed files with 93 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/external/blizzard-database-library vendored

Submodule src/external/blizzard-database-library updated: 1cefbe958a...260564ee1f

183
src/noggit/sql/ClientDatabase.cpp
View File

@@ -9,6 +9,12 @@
namespace Noggit
{
// namespace Sql
auto escapeSqlString = [](const QString& str) -> QString {
QString result = str;
result.replace("'", "''"); // double single quotes for SQL
return "'" + result + "'";
};
std::optional<Structures::BlizzardDatabaseRow> ClientDatabase::getRowById(const std::string& tableName, unsigned int id)
@@ -28,7 +34,7 @@ namespace Noggit
return row;
}
bool ClientDatabase::testUploadDBCtoDB(BlizzardDatabaseLib::BlizzardDatabaseTable& table)
bool ClientDatabase::testUploadDBCtoDB(const BlizzardDatabaseLib::BlizzardDatabaseTable& table)
{
auto& db_mgr = Noggit::Sql::DatabaseManager::instance();
@@ -95,7 +101,6 @@ namespace Noggit
// insert if fresh_table, otherwise replace?
// TODOOOOOOOOOOOOOOOOOOOOOOO : fill db with data
auto row_definition = table.GetRecordDefinition();
auto sql_record_format = recordFormat(table_name);
@@ -106,139 +111,109 @@ namespace Noggit
{
column_names.append(sql_column_format.Name.c_str());
}
// INSERT INTO table (col1, col2, ...) VALUES (?, ?, ...)
QString sql = QString("INSERT INTO `%1` (%2) VALUES (%3)")
.arg(sql_table_name)
.arg(column_names.join(", "))
.arg(QString("?, ").repeated(column_names.size()).chopped(2));
QSqlQuery query(noggit_db);
if (!query.prepare(sql))
{
qWarning() << "Prepare failed:" << query.lastError().text();
return false;
}
int colCount = column_names.size();
QElapsedTimer timer;
timer.start();
// using transaction to speed up bulk query
// noggit_db.transaction();
const int batchSize = 2000;
int rowCount = 0;
int currentSize = 0;
// batching:
// One QVariantList per column
std::vector<QVariantList> columnData(column_names.size());
noggit_db.transaction();
QStringList rowBuffer; // holds each row as a string
rowBuffer.reserve(batchSize); // reserve for batch size
while (client_table_iterator.HasRecords())
{
auto& record = client_table_iterator.Next();
int colIndex = 0;
QStringList colValues;
colValues.reserve(column_names.size()); // reserve columns per row
for (auto& column_def : row_definition.ColumnDefinitions)
{
if (column_def.Type == "int" && column_def.isID) // id column isn't saved in the map
if (column_def.Type == "int" && column_def.isID)
{
// query.addBindValue(record.RecordId);
columnData[colIndex++].append(record.RecordId); // batching
colValues.append(QString::number(record.RecordId));
continue;
}
auto& rowColumn = record.Columns.at(column_def.Name);
if (column_def.Type == "int")
if (column_def.Type == "locstring")
{
if (column_def.arrLength > 1)
{
for (int i = 0; i < column_def.arrLength; i++)
{
// int Value = std::stoi(rowColumn.Values[i]);
// query.addBindValue(QString::fromStdString(rowColumn.Values[i]).toInt());
columnData[colIndex++].append(QString::fromStdString(rowColumn.Value).toInt());
}
}
else
{
// int Value = std::stoi(rowColumn.Value);
// query.addBindValue(QString::fromStdString(rowColumn.Value).toInt());
columnData[colIndex++].append(QString::fromStdString(rowColumn.Value).toInt());
}
}
else if (column_def.Type == "float")
{
if (column_def.arrLength > 1)
{
for (int i = 0; i < column_def.arrLength; i++)
{
// float Value = std::stof(rowColumn.Values[i]);
// query.addBindValue(QString::fromStdString(rowColumn.Values[i]).toFloat());
columnData[colIndex++].append(QString::fromStdString(rowColumn.Values[i]).toFloat());
}
}
else
{
// float Value = std::stof(rowColumn.Value);
// query.addBindValue(QString::fromStdString(rowColumn.Value).toFloat());
columnData[colIndex++].append(QString::fromStdString(rowColumn.Value).toFloat());
}
}
else if (column_def.Type == "string")
{
if (column_def.arrLength > 1)
{
for (int i = 0; i < column_def.arrLength; i++)
{
// std::string value = rowColumn.Values[i];
// query.addBindValue(QString::fromStdString(rowColumn.Values[i]));
columnData[colIndex++].append(QString::fromStdString(rowColumn.Values[i]));
}
}
else
{
// std::string Value = rowColumn.Value;
// query.addBindValue(QString::fromStdString(rowColumn.Value));
columnData[colIndex++].append(QString::fromStdString(rowColumn.Value));
}
}
else if (column_def.Type == "locstring")
{
for (int i = 0; i < 16; i++)
{
// query.addBindValue(QString::fromStdString(rowColumn.Values[i]));
columnData[colIndex++].append(QString::fromStdString(rowColumn.Values[i]));
}
for (int i = 0; i < 16; ++i)
colValues.append(escapeSqlString(QString::fromStdString(rowColumn.Values[i])));
auto& flagValue = record.Columns.at(column_def.Name + "_flags");
// query.addBindValue(QString::fromStdString(flagValue.Value).toInt());
columnData[colIndex++].append(QString::fromStdString(flagValue.Value).toInt());
auto& flagValue = record.Columns.at(column_def.Name + "_flags").Value;
colValues.append(QString::fromStdString(flagValue));
}
else
{
int len = (column_def.arrLength > 1) ? column_def.arrLength : 1;
for (int i = 0; i < len; ++i)
{
if (column_def.Type == "string")
colValues.append(escapeSqlString((len > 1) ? QString::fromStdString(rowColumn.Values[i])
: QString::fromStdString(rowColumn.Value)));
else // int/float
colValues.append((len > 1) ? QString::fromStdString(rowColumn.Values[i])
: QString::fromStdString(rowColumn.Value));
}
}
}
// if (!query.exec())
// {
// qWarning() << "Insert failed:" << query.lastError().text();
// noggit_db.rollback();
// return false;
// }
currentSize += colValues.size();
// append row as a single string
rowBuffer.append("(" + colValues.join(", ") + ")");
rowCount++;
// flush batch
if (rowCount % batchSize == 0)
{
QString sql;
sql.reserve(1024 * 64); // reserve ~64KB, adjust if rows are big
sql = QString("INSERT INTO `%1` (%2) VALUES ")
.arg(sql_table_name)
.arg(column_names.join(", "));
sql += rowBuffer.join(", ");
QSqlQuery query(noggit_db);
if (!query.exec(sql))
{
qWarning() << "Batch insert failed:" << query.lastError().text();
noggit_db.rollback();
return false;
}
rowBuffer.clear();
}
}
// Bind all columnData at once
for (auto& col : columnData)
query.addBindValue(col);
noggit_db.transaction();
if (!query.execBatch(QSqlQuery::ValuesAsColumns))
// flush remaining rows
if (!rowBuffer.isEmpty())
{
qWarning() << "Batch insert failed:" << query.lastError().text();
noggit_db.rollback();
return false;
QString sql;
sql.reserve(1024 * 64); // 64kb
sql = QString("INSERT INTO `%1` (%2) VALUES ")
.arg(sql_table_name)
.arg(column_names.join(", "));
sql += rowBuffer.join(", ");
QSqlQuery query(noggit_db);
if (!query.exec(sql))
{
qWarning() << "Final batch insert failed:" << query.lastError().text();
noggit_db.rollback();
return false;
}
}
noggit_db.commit();
// benchmark
qint64 elapsedMs = timer.elapsed();
qDebug() << "Inserted" << table.RecordCount() << "rows in"
<< elapsedMs << "ms ("
<< (table.RecordCount() * 1000.0 / elapsedMs) << " rows/sec)";

2
src/noggit/sql/ClientDatabase.h
View File

@@ -39,7 +39,7 @@ namespace Noggit
static std::optional<Structures::BlizzardDatabaseRow> getRowById(const std::string& tableName, unsigned int id); // constructs a row either from db or client
static bool testUploadDBCtoDB(BlizzardDatabaseLib::BlizzardDatabaseTable& table);
static bool testUploadDBCtoDB(const BlizzardDatabaseLib::BlizzardDatabaseTable& table);
static void TODODeploySqlToClient();

18
src/noggit/ui/tools/MapCreationWizard/Ui/MapCreationWizard.cpp
View File

@@ -85,7 +85,7 @@ MapCreationWizard::MapCreationWizard(std::shared_ptr<Project::NoggitProject> pro
// Fill selector combo
const auto& table = std::string("Map");
auto mapTable = _project->ClientDatabase->LoadTable(table, readFileAsIMemStream);
auto& mapTable = _project->ClientDatabase->LoadTable(table, readFileAsIMemStream);
int count = 0;
auto iterator = mapTable.Records();
@@ -630,19 +630,21 @@ void MapCreationWizard::selectMap(int map_id)
// int map_id = world->getMapID();
auto table = _project->ClientDatabase->LoadTable("Map", readFileAsIMemStream);
auto& table = _project->ClientDatabase->LoadTable("Map", readFileAsIMemStream);
auto record = table.RecordById(map_id);
/// test area, delete later
// auto table = _project->ClientDatabase->GetTable("Map");
/// test area, delete later /////////////
QSettings settings;
bool use_mysql = settings.value("project/mysql/enabled", false).toBool();
if (use_mysql)
{
// bool valid_conn = mysql::testConnection(true);
Noggit::ClientDatabase::testUploadDBCtoDB(table);
auto& test_table = _project->ClientDatabase->LoadTable("ItemDisplayInfo", readFileAsIMemStream);
Noggit::ClientDatabase::testUploadDBCtoDB(test_table);
// TODO : crashes if not unloading
//_project->ClientDatabase->UnloadTable("AreaTable");
}
///////////////////////////////
@@ -741,7 +743,7 @@ void MapCreationWizard::selectMap(int map_id)
_project->ClientDatabase->UnloadTable("Map");
auto difficulty_table = _project->ClientDatabase->LoadTable("MapDifficulty", readFileAsIMemStream);
auto& difficulty_table = _project->ClientDatabase->LoadTable("MapDifficulty", readFileAsIMemStream);
auto iterator = difficulty_table.Records();
@@ -778,7 +780,7 @@ void MapCreationWizard::selectMapDifficulty()
if (!selected_difficulty_id)
return;
auto difficulty_table = _project->ClientDatabase->LoadTable("MapDifficulty", readFileAsIMemStream);
auto& difficulty_table = _project->ClientDatabase->LoadTable("MapDifficulty", readFileAsIMemStream);
auto record = difficulty_table.RecordById(selected_difficulty_id);
//_difficulty_type;

2
src/noggit/ui/tools/PresetEditor/Ui/PresetEditor.cpp
View File

@@ -85,7 +85,7 @@ PresetEditorWidget::PresetEditorWidget(std::shared_ptr<Project::NoggitProject> p
ui->worldSelector->setItemData(0, QVariant(-1));
const auto& table = std::string("Map");
auto mapTable = _project->ClientDatabase->LoadTable(table, readFileAsIMemStream);
auto& mapTable = _project->ClientDatabase->LoadTable(table, readFileAsIMemStream);
int count = 1;
auto iterator = mapTable.Records();

2
src/noggit/ui/windows/noggitWindow/components/BuildMapListComponent.cpp
View File

@@ -25,7 +25,7 @@ void BuildMapListComponent::buildMapList(Noggit::Ui::Windows::NoggitWindow* pare
}
const auto& table = std::string("Map");
auto map_table = parent->_project->ClientDatabase->LoadTable(table, readFileAsIMemStream);
auto& map_table = parent->_project->ClientDatabase->LoadTable(table, readFileAsIMemStream);
auto iterator = map_table.Records();
auto pinned_maps = std::vector<Widget::MapListData>();