diff --git a/src/external/blizzard-database-library b/src/external/blizzard-database-library index 1cefbe95..260564ee 160000 --- a/src/external/blizzard-database-library +++ b/src/external/blizzard-database-library @@ -1 +1 @@ -Subproject commit 1cefbe958a765cf71d42c80182aaae1d50baacf5 +Subproject commit 260564ee1f379b393959477867855ff35f3b63d6 diff --git a/src/noggit/sql/ClientDatabase.cpp b/src/noggit/sql/ClientDatabase.cpp index 9d60f9bf..ffa84b38 100644 --- a/src/noggit/sql/ClientDatabase.cpp +++ b/src/noggit/sql/ClientDatabase.cpp @@ -9,6 +9,12 @@ namespace Noggit { + // namespace Sql + auto escapeSqlString = [](const QString& str) -> QString { + QString result = str; + result.replace("'", "''"); // double single quotes for SQL + return "'" + result + "'"; + }; std::optional ClientDatabase::getRowById(const std::string& tableName, unsigned int id) @@ -28,7 +34,7 @@ namespace Noggit return row; } - bool ClientDatabase::testUploadDBCtoDB(BlizzardDatabaseLib::BlizzardDatabaseTable& table) + bool ClientDatabase::testUploadDBCtoDB(const BlizzardDatabaseLib::BlizzardDatabaseTable& table) { auto& db_mgr = Noggit::Sql::DatabaseManager::instance(); @@ -95,7 +101,6 @@ namespace Noggit // insert if fresh_table, otherwise replace? - // TODOOOOOOOOOOOOOOOOOOOOOOO : fill db with data auto row_definition = table.GetRecordDefinition(); auto sql_record_format = recordFormat(table_name); @@ -106,139 +111,109 @@ namespace Noggit { column_names.append(sql_column_format.Name.c_str()); } - - // INSERT INTO table (col1, col2, ...) VALUES (?, ?, ...) - QString sql = QString("INSERT INTO `%1` (%2) VALUES (%3)") - .arg(sql_table_name) - .arg(column_names.join(", ")) - .arg(QString("?, ").repeated(column_names.size()).chopped(2)); - - QSqlQuery query(noggit_db); - if (!query.prepare(sql)) - { - qWarning() << "Prepare failed:" << query.lastError().text(); - return false; - } + int colCount = column_names.size(); QElapsedTimer timer; timer.start(); - // using transaction to speed up bulk query - // noggit_db.transaction(); + const int batchSize = 2000; + int rowCount = 0; + int currentSize = 0; - // batching: - // One QVariantList per column - std::vector columnData(column_names.size()); + noggit_db.transaction(); + QStringList rowBuffer; // holds each row as a string + rowBuffer.reserve(batchSize); // reserve for batch size while (client_table_iterator.HasRecords()) { auto& record = client_table_iterator.Next(); - int colIndex = 0; + QStringList colValues; + colValues.reserve(column_names.size()); // reserve columns per row for (auto& column_def : row_definition.ColumnDefinitions) { - if (column_def.Type == "int" && column_def.isID) // id column isn't saved in the map + if (column_def.Type == "int" && column_def.isID) { - // query.addBindValue(record.RecordId); - columnData[colIndex++].append(record.RecordId); // batching + colValues.append(QString::number(record.RecordId)); continue; } auto& rowColumn = record.Columns.at(column_def.Name); - if (column_def.Type == "int") + if (column_def.Type == "locstring") { - if (column_def.arrLength > 1) - { - for (int i = 0; i < column_def.arrLength; i++) - { - // int Value = std::stoi(rowColumn.Values[i]); - // query.addBindValue(QString::fromStdString(rowColumn.Values[i]).toInt()); - columnData[colIndex++].append(QString::fromStdString(rowColumn.Value).toInt()); - } - } - else - { - // int Value = std::stoi(rowColumn.Value); - // query.addBindValue(QString::fromStdString(rowColumn.Value).toInt()); - columnData[colIndex++].append(QString::fromStdString(rowColumn.Value).toInt()); - } - } - else if (column_def.Type == "float") - { - if (column_def.arrLength > 1) - { - for (int i = 0; i < column_def.arrLength; i++) - { - // float Value = std::stof(rowColumn.Values[i]); - // query.addBindValue(QString::fromStdString(rowColumn.Values[i]).toFloat()); - columnData[colIndex++].append(QString::fromStdString(rowColumn.Values[i]).toFloat()); - } - } - else - { - // float Value = std::stof(rowColumn.Value); - // query.addBindValue(QString::fromStdString(rowColumn.Value).toFloat()); - columnData[colIndex++].append(QString::fromStdString(rowColumn.Value).toFloat()); - } - } - else if (column_def.Type == "string") - { - if (column_def.arrLength > 1) - { - for (int i = 0; i < column_def.arrLength; i++) - { - // std::string value = rowColumn.Values[i]; - // query.addBindValue(QString::fromStdString(rowColumn.Values[i])); - columnData[colIndex++].append(QString::fromStdString(rowColumn.Values[i])); - } - } - else - { - // std::string Value = rowColumn.Value; - // query.addBindValue(QString::fromStdString(rowColumn.Value)); - columnData[colIndex++].append(QString::fromStdString(rowColumn.Value)); - } - } - else if (column_def.Type == "locstring") - { - for (int i = 0; i < 16; i++) - { - // query.addBindValue(QString::fromStdString(rowColumn.Values[i])); - columnData[colIndex++].append(QString::fromStdString(rowColumn.Values[i])); - } + for (int i = 0; i < 16; ++i) + colValues.append(escapeSqlString(QString::fromStdString(rowColumn.Values[i]))); - auto& flagValue = record.Columns.at(column_def.Name + "_flags"); - // query.addBindValue(QString::fromStdString(flagValue.Value).toInt()); - columnData[colIndex++].append(QString::fromStdString(flagValue.Value).toInt()); + auto& flagValue = record.Columns.at(column_def.Name + "_flags").Value; + colValues.append(QString::fromStdString(flagValue)); + } + else + { + int len = (column_def.arrLength > 1) ? column_def.arrLength : 1; + for (int i = 0; i < len; ++i) + { + if (column_def.Type == "string") + colValues.append(escapeSqlString((len > 1) ? QString::fromStdString(rowColumn.Values[i]) + : QString::fromStdString(rowColumn.Value))); + else // int/float + colValues.append((len > 1) ? QString::fromStdString(rowColumn.Values[i]) + : QString::fromStdString(rowColumn.Value)); + } } } - // if (!query.exec()) - // { - // qWarning() << "Insert failed:" << query.lastError().text(); - // noggit_db.rollback(); - // return false; - // } + + currentSize += colValues.size(); + + // append row as a single string + rowBuffer.append("(" + colValues.join(", ") + ")"); + rowCount++; + + // flush batch + if (rowCount % batchSize == 0) + { + QString sql; + sql.reserve(1024 * 64); // reserve ~64KB, adjust if rows are big + sql = QString("INSERT INTO `%1` (%2) VALUES ") + .arg(sql_table_name) + .arg(column_names.join(", ")); + sql += rowBuffer.join(", "); + + QSqlQuery query(noggit_db); + if (!query.exec(sql)) + { + qWarning() << "Batch insert failed:" << query.lastError().text(); + noggit_db.rollback(); + return false; + } + rowBuffer.clear(); + } } - // Bind all columnData at once - for (auto& col : columnData) - query.addBindValue(col); - - noggit_db.transaction(); - - if (!query.execBatch(QSqlQuery::ValuesAsColumns)) + // flush remaining rows + if (!rowBuffer.isEmpty()) { - qWarning() << "Batch insert failed:" << query.lastError().text(); - noggit_db.rollback(); - return false; + QString sql; + sql.reserve(1024 * 64); // 64kb + sql = QString("INSERT INTO `%1` (%2) VALUES ") + .arg(sql_table_name) + .arg(column_names.join(", ")); + sql += rowBuffer.join(", "); + + QSqlQuery query(noggit_db); + if (!query.exec(sql)) + { + qWarning() << "Final batch insert failed:" << query.lastError().text(); + noggit_db.rollback(); + return false; + } } noggit_db.commit(); + // benchmark qint64 elapsedMs = timer.elapsed(); - qDebug() << "Inserted" << table.RecordCount() << "rows in" << elapsedMs << "ms (" << (table.RecordCount() * 1000.0 / elapsedMs) << " rows/sec)"; diff --git a/src/noggit/sql/ClientDatabase.h b/src/noggit/sql/ClientDatabase.h index f6ae4fd0..dace359b 100644 --- a/src/noggit/sql/ClientDatabase.h +++ b/src/noggit/sql/ClientDatabase.h @@ -39,7 +39,7 @@ namespace Noggit static std::optional getRowById(const std::string& tableName, unsigned int id); // constructs a row either from db or client - static bool testUploadDBCtoDB(BlizzardDatabaseLib::BlizzardDatabaseTable& table); + static bool testUploadDBCtoDB(const BlizzardDatabaseLib::BlizzardDatabaseTable& table); static void TODODeploySqlToClient(); diff --git a/src/noggit/ui/tools/MapCreationWizard/Ui/MapCreationWizard.cpp b/src/noggit/ui/tools/MapCreationWizard/Ui/MapCreationWizard.cpp index 8506c4a3..bfca1de9 100755 --- a/src/noggit/ui/tools/MapCreationWizard/Ui/MapCreationWizard.cpp +++ b/src/noggit/ui/tools/MapCreationWizard/Ui/MapCreationWizard.cpp @@ -85,7 +85,7 @@ MapCreationWizard::MapCreationWizard(std::shared_ptr pro // Fill selector combo const auto& table = std::string("Map"); - auto mapTable = _project->ClientDatabase->LoadTable(table, readFileAsIMemStream); + auto& mapTable = _project->ClientDatabase->LoadTable(table, readFileAsIMemStream); int count = 0; auto iterator = mapTable.Records(); @@ -630,19 +630,21 @@ void MapCreationWizard::selectMap(int map_id) // int map_id = world->getMapID(); - auto table = _project->ClientDatabase->LoadTable("Map", readFileAsIMemStream); + auto& table = _project->ClientDatabase->LoadTable("Map", readFileAsIMemStream); auto record = table.RecordById(map_id); - /// test area, delete later - // auto table = _project->ClientDatabase->GetTable("Map"); + /// test area, delete later ///////////// QSettings settings; - bool use_mysql = settings.value("project/mysql/enabled", false).toBool(); if (use_mysql) { // bool valid_conn = mysql::testConnection(true); - Noggit::ClientDatabase::testUploadDBCtoDB(table); + auto& test_table = _project->ClientDatabase->LoadTable("ItemDisplayInfo", readFileAsIMemStream); + Noggit::ClientDatabase::testUploadDBCtoDB(test_table); + + // TODO : crashes if not unloading + //_project->ClientDatabase->UnloadTable("AreaTable"); } /////////////////////////////// @@ -741,7 +743,7 @@ void MapCreationWizard::selectMap(int map_id) _project->ClientDatabase->UnloadTable("Map"); - auto difficulty_table = _project->ClientDatabase->LoadTable("MapDifficulty", readFileAsIMemStream); + auto& difficulty_table = _project->ClientDatabase->LoadTable("MapDifficulty", readFileAsIMemStream); auto iterator = difficulty_table.Records(); @@ -778,7 +780,7 @@ void MapCreationWizard::selectMapDifficulty() if (!selected_difficulty_id) return; - auto difficulty_table = _project->ClientDatabase->LoadTable("MapDifficulty", readFileAsIMemStream); + auto& difficulty_table = _project->ClientDatabase->LoadTable("MapDifficulty", readFileAsIMemStream); auto record = difficulty_table.RecordById(selected_difficulty_id); //_difficulty_type; diff --git a/src/noggit/ui/tools/PresetEditor/Ui/PresetEditor.cpp b/src/noggit/ui/tools/PresetEditor/Ui/PresetEditor.cpp index 8fb7300c..73a35b60 100755 --- a/src/noggit/ui/tools/PresetEditor/Ui/PresetEditor.cpp +++ b/src/noggit/ui/tools/PresetEditor/Ui/PresetEditor.cpp @@ -85,7 +85,7 @@ PresetEditorWidget::PresetEditorWidget(std::shared_ptr p ui->worldSelector->setItemData(0, QVariant(-1)); const auto& table = std::string("Map"); - auto mapTable = _project->ClientDatabase->LoadTable(table, readFileAsIMemStream); + auto& mapTable = _project->ClientDatabase->LoadTable(table, readFileAsIMemStream); int count = 1; auto iterator = mapTable.Records(); diff --git a/src/noggit/ui/windows/noggitWindow/components/BuildMapListComponent.cpp b/src/noggit/ui/windows/noggitWindow/components/BuildMapListComponent.cpp index 16bb634c..4660f38c 100755 --- a/src/noggit/ui/windows/noggitWindow/components/BuildMapListComponent.cpp +++ b/src/noggit/ui/windows/noggitWindow/components/BuildMapListComponent.cpp @@ -25,7 +25,7 @@ void BuildMapListComponent::buildMapList(Noggit::Ui::Windows::NoggitWindow* pare } const auto& table = std::string("Map"); - auto map_table = parent->_project->ClientDatabase->LoadTable(table, readFileAsIMemStream); + auto& map_table = parent->_project->ClientDatabase->LoadTable(table, readFileAsIMemStream); auto iterator = map_table.Records(); auto pinned_maps = std::vector();