Fix color calculation for variable 'b' in WMO.cpp

cherry pick from 0347333f95
optimize alphamap updates
2025-10-23 23:34:54 +02:00 · 2025-10-16 22:50:07 +02:00 · 2025-09-24 05:04:11 +02:00 · 2025-09-23 05:48:37 +02:00 · 2025-09-23 01:31:41 +02:00 · 2025-09-19 21:37:39 +02:00
24 changed files with 1551 additions and 337 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -1,18 +1,7 @@
-/**
-!/bin/**
-!/cmake/**
-!/etc/**
-!/include/**
-!/media/**
-!/resources/**
-!/sql/**
-!/src/**
-!/test/**
-!/.idea/**
-!.gitignore
-!.CMakeLists.txt
-!COPYING
-!Doxyfile
-!README.md
-!todo_1.4.md
-!uid_fix_concept.md
+
+/out
+/bin
+/build
+/.vs
+/CMakePresets.json
+/CMakeUserPresets.json
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -88,7 +88,6 @@ SET(LIBARY_OUTPUT_PATH bin)
 SET(EXTERNAL_SOURCE_DIR src/external)
 set_property(GLOBAL PROPERTY USE_FOLDERS ON)

-# OPTION(USE_SQL "Enable sql uid save ? (require mysql installed)" OFF)
 OPTION(VALIDATE_OPENGL_PROGRAMS "Validate Opengl programs" OFF)

 IF(VALIDATE_OPENGL_PROGRAMS)
@@ -115,32 +114,7 @@ FetchContent_Declare(
 FetchContent_MakeAvailable(FastNoise2)

 FIND_PACKAGE(Sol2 REQUIRED)
-FIND_PACKAGE(Qt5 COMPONENTS Widgets OpenGLExtensions Gui Network Xml Multimedia REQUIRED)
-
-# IF(USE_SQL)
-  FIND_LIBRARY(MYSQL_LIBRARY NAMES libmysql
-              HINTS "${CMAKE_SOURCE_DIR}/../Noggit3libs/mysql")
-  FIND_LIBRARY(MYSQLCPPCONN_LIBRARY NAMES mysqlcppconn
-              HINTS "${CMAKE_SOURCE_DIR}/../Noggit3libs/mysql/connector")
-  FIND_PATH(MYSQLCPPCONN_INCLUDE NAMES cppconn/driver.h
-              HINTS "${CMAKE_SOURCE_DIR}/../Noggit3libs/mysql/connector")
-
-  # Ensure we always include the parent of 'cppconn' folder
-  if(EXISTS "${MYSQLCPPCONN_INCLUDE}/cppconn")
-    set(MYSQLCPPCONN_INCLUDE "${MYSQLCPPCONN_INCLUDE}")
-  elseif(EXISTS "${MYSQLCPPCONN_INCLUDE}/driver.h")
-    get_filename_component(MYSQLCPPCONN_INCLUDE "${MYSQLCPPCONN_INCLUDE}" DIRECTORY)
-  endif()
-
-  IF(MYSQL_LIBRARY AND MYSQLCPPCONN_LIBRARY AND MYSQLCPPCONN_INCLUDE)
-    ADD_DEFINITIONS(-DUSE_MYSQL_UID_STORAGE)
-    SET (mysql_sources src/mysql/mysql.cpp)
-    SET (mysql_headers src/mysql/mysql.h)
-    SOURCE_GROUP("mysql"  FILES ${mysql_sources} ${mysql_headers})
-  ELSE()
-    MESSAGE(FATAL_ERROR "MySQL lib or connector not found")
-  ENDIF()
-# ENDIF()
+FIND_PACKAGE(Qt5 COMPONENTS Widgets OpenGLExtensions Gui Network Xml Multimedia Sql REQUIRED)

 ADD_SUBDIRECTORY("${EXTERNAL_SOURCE_DIR}/qt-color-widgets")
 ADD_SUBDIRECTORY("${EXTERNAL_SOURCE_DIR}/framelesshelper")
@@ -297,7 +271,6 @@ ADD_EXECUTABLE(noggit
  ${opengl_sources}
  ${math_sources}
  ${external_sources}
-  ${mysql_sources}
  ${os_sources}
  ${util_sources}
  ${util_headers}
@@ -311,7 +284,6 @@ ADD_EXECUTABLE(noggit
  ${opengl_headers}
  ${math_headers}
  ${external_headers}
-  ${mysql_headers}
  ${os_headers}
  ${png_blp_headers}
  ${ResFiles}
@@ -355,6 +327,7 @@ TARGET_LINK_LIBRARIES (noggit
  Qt5::Xml
  Qt5::Network
  Qt5::Multimedia
+  Qt5::Sql
  ColorWidgets-qt5
  FramelessHelper
  qt_imgui_widgets
@@ -371,6 +344,78 @@ TARGET_LINK_LIBRARIES (noggit
  blizzard-database-library
  rapidfuzz::rapidfuzz
 )
+include(ExternalProject)
+
+
+# Fetch qsqlmysql.dll plugin based on Qt version #####
+execute_process(
+    COMMAND ${Qt5Core_DIR}/../../../bin/qmake -query QT_VERSION
+    OUTPUT_VARIABLE QT_FULL_VERSION
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+)
+message(STATUS "Qt full version: ${QT_FULL_VERSION}")
+
+# prebuilds available at : https://github.com/thecodemonkey86/qt_mysql_driver/releases
+# get direct download links
+if(WIN32)
+  if(MSVC)
+        set(QMYSQL_PREBUILT_DRIVER_URL
+            "https://github.com/thecodemonkey86/qt_mysql_driver/files/5575770/qsqlmysql.dll_Qt_SQL_driver_${QT_FULL_VERSION}_MSVC2019_64-bit.zip"
+        )
+    elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") # MinGW
+        set(QMYSQL_PREBUILT_DRIVER_URL
+            "https://github.com/thecodemonkey86/qt_mysql_driver/files/5575769/qsqlmysql.dll_Qt_SQL_driver_${QT_FULL_VERSION}_MinGW_8.1.0_64-bit.zip"
+        )
+    else()
+        message(WARNING "Unsupported Windows compiler for prebuilt MySQL Qt driver")
+  endif()
+elseif(UNIX)
+  set(QMYSQL_PREBUILT_DRIVER_URL
+      "https://github.com/thecodemonkey86/qt_mysql_driver/files/6388052/libqsqlmysql.so_Qt_SQL_driver_${QT_FULL_VERSION}_gcc64.zip"
+)
+else()
+  message(WARNING "Unsupported platform for prebuilt MySQL Qt driver")
+endif()
+
+set(QMYSQL_PREBUILT_DIR "${CMAKE_BINARY_DIR}/_deps/qt_mysql_driver")
+set(QMYSQL_EXTRACT_DIR "${QMYSQL_PREBUILT_DIR}/sqldrivers")
+set(QMYSQL_ZIP "${QMYSQL_PREBUILT_DIR}/qsqlmysql_prebuilt.zip")
+
+# Download zip only if not already present
+if(NOT EXISTS "${QMYSQL_ZIP}")
+  message(STATUS "qt MySQL DRIVER zip not found. Downloading qmysql from : ${QMYSQL_PREBUILT_DRIVER_URL}")
+
+  file(DOWNLOAD
+      ${QMYSQL_PREBUILT_DRIVER_URL}
+      ${QMYSQL_ZIP}
+      # SHOW_PROGRESS
+      STATUS DOWNLOAD_STATUS
+      TLS_VERIFY ON
+  )
+
+  list(GET DOWNLOAD_STATUS 0 STATUS_CODE)
+  if(NOT STATUS_CODE EQUAL 0)
+      message(WARNING "Failed to download prebuilt qt MySQL driver: ${DOWNLOAD_STATUS}")
+      message(WARNING "You can manually download it from https://github.com/thecodemonkey86/qt_mysql_driver/releases or build qsqlmysql.dll yourself from QT source.")
+  endif()
+else()
+    message(STATUS "Prebuilt MySQL Qt driver zip already exists, skipping download")
+endif()
+
+# Extract zip
+file(ARCHIVE_EXTRACT
+    INPUT ${QMYSQL_ZIP}
+    DESTINATION ${QMYSQL_EXTRACT_DIR}
+)
+message(STATUS "Extracted prebuilt MySQL Qt driver at ${QMYSQL_EXTRACT_DIR}. Deps will be deploeyd after build.")
+
+  
+# deploy
+add_custom_command(TARGET noggit POST_BUILD
+COMMAND ${CMAKE_COMMAND} -E copy_directory
+    "${QMYSQL_EXTRACT_DIR}"
+    "$<TARGET_FILE_DIR:noggit>"
+)

 #add distribution themes
 add_custom_command(TARGET noggit POST_BUILD
@@ -452,11 +497,6 @@ IF(APPLE)
  TARGET_LINK_LIBRARIES (noggit "-framework Cocoa" "-framework AppKit" "-framework Foundation")
 ENDIF()

-IF(MYSQL_LIBRARY AND MYSQLCPPCONN_LIBRARY AND MYSQLCPPCONN_INCLUDE)
-  TARGET_LINK_LIBRARIES(noggit ${MYSQL_LIBRARY} ${MYSQLCPPCONN_LIBRARY})
-  TARGET_INCLUDE_DIRECTORIES(noggit SYSTEM PRIVATE ${MYSQLCPPCONN_INCLUDE})
-ENDIF()
-
 IF(NOGGIT_LOGTOCONSOLE AND WIN32)
  SET_PROPERTY(TARGET noggit APPEND PROPERTY LINK_FLAGS_DEBUG "/SUBSYSTEM:CONSOLE")
  SET_PROPERTY(TARGET noggit APPEND PROPERTY COMPILE_DEFINITIONS $<$<CONFIG:Debug>:"_CONSOLE">)
--- a/src/external/blizzard-database-library
+++ b/src/external/blizzard-database-library
--- a/src/mysql/mysql.cpp
+++ b/src/mysql/mysql.cpp
@@ -1,166 +0,0 @@
-// This file is part of Noggit3, licensed under GNU General Public License (version 3).
-
-#include <mysql/mysql.h>
-#include <noggit/world.h>
-
-#include <QSettings>
-#include <QMessageBox>
-
-#include <cppconn/driver.h>
-#include <cppconn/prepared_statement.h>
-#include <cppconn/exception.h>
-
-namespace
-{
-  std::unique_ptr<sql::Connection> connect()
-  {
-		QSettings settings;
-
-		// if using release SQL binaries in debug mode it will crash https://bugs.mysql.com/bug.php?id=91238 unless using sql strings
-		// tcp://127.0.0.1:3306
-		const sql::SQLString hostname = "tcp://" + settings.value("project/mysql/server").toString().toStdString() + ":" + settings.value("project/mysql/port", "3306").toString().toStdString();
-		const sql::SQLString userName = settings.value("project/mysql/user").toString().toStdString();
-		const sql::SQLString password = settings.value("project/mysql/pwd").toString().toStdString();
-		const sql::SQLString schema = settings.value("project/mysql/db").toString().toStdString();
-
-		try
-		{
-			std::unique_ptr<sql::Connection> Con(get_driver_instance()->connect(hostname, userName, password));
-
-			// crete database if it doesn't exist
-			std::string createdb_statement = "CREATE DATABASE IF NOT EXISTS " + schema;
-			std::unique_ptr<sql::PreparedStatement> dbpstmt(Con->prepareStatement(createdb_statement));
-			std::unique_ptr<sql::ResultSet> res(dbpstmt->executeQuery());
-			
-			Con->setSchema(schema);
-
-			// create table if it doesn't exist, querries from src/sql
-			std::unique_ptr<sql::PreparedStatement> tablepstmt(Con->prepareStatement("CREATE TABLE IF NOT EXISTS `UIDs` ("
-												"`_map_id` int(11) NOT NULL,"
-												"`UID` int(11) NOT NULL,"
-												"PRIMARY KEY(`_map_id`)"
-												") ENGINE = InnoDB DEFAULT CHARSET = latin1;"));
-			std::unique_ptr<sql::ResultSet> tableres(tablepstmt->executeQuery());
-
-			return Con;
-		}
-		catch (sql::SQLException& e)
-		{
-
-			return nullptr;
-		}
-		catch (std::exception& e)
-		{
-			std::cerr << "SQL Other exception: " << e.what() << std::endl;
-			return nullptr;
-		}
-  }
-}
-
-namespace mysql
-{
-  bool testConnection(bool report_only_err)
-  {
-	  QSettings settings;
-	  // if using release SQL binaries in debug mode it will crash https://bugs.mysql.com/bug.php?id=91238 unless using sql strings
-	  const sql::SQLString hostname = "tcp://" + settings.value("project/mysql/server").toString().toStdString() + ":" + settings.value("project/mysql/port", "3306").toString().toStdString();
-	  const sql::SQLString userName = settings.value("project/mysql/user").toString().toStdString();
-	  const sql::SQLString password = settings.value("project/mysql/pwd").toString().toStdString();
-
-	  QMessageBox prompt;
-	  prompt.setWindowFlag(Qt::WindowStaysOnTopHint);
-	  try
-	  {
-			sql::Driver* driver = get_driver_instance();
-			sql::Connection* connection = driver->connect(hostname, userName, password);
-			std::unique_ptr<sql::Connection> Con(connection);
-
-		  prompt.setIcon(QMessageBox::Information);
-		  prompt.setText("Succesfully connected to MySQL database.");
-		  prompt.setWindowTitle("Success");
-
-		  if (!report_only_err)
-			prompt.exec();
-
-		  return true;
-	  }
-	  catch (sql::SQLException& e)
-	  {
-		  
-		  prompt.setIcon(QMessageBox::Warning);
-		  prompt.setText("Failed to load MySQL database, check your settings. \nIf you did not intend to use this feature, disable it in Noggit->settings->MySQL");
-		  prompt.setWindowTitle("Noggit Database Error");
-		  // disable if connection is not valid
-		  // settings.value("project/mysql/enabled") = false;
-		  std::stringstream promptText;
-
-		  promptText << "\n# ERR: " << e.what();
-		  promptText << "\n (MySQL error code: " << e.getErrorCode() + ")";
-
-		  prompt.setInformativeText(promptText.str().c_str());
-		  prompt.exec();
-
-		  return false;
-	  }
-		catch (std::exception& e)
-		{
-			std::cerr << "SQL Other exception: " << e.what() << std::endl;
-			return false;
-		}
-  }
-
-  bool hasMaxUIDStoredDB(std::size_t mapID)
-  {
-	  auto Con(connect());
-	  if (Con == nullptr)
-		  return false;
-	  std::unique_ptr<sql::PreparedStatement> pstmt(Con->prepareStatement("SELECT * FROM `UIDs` WHERE `_map_id`=(?)"));
-	  pstmt->setInt(1, mapID);
-	  std::unique_ptr<sql::ResultSet> res(pstmt->executeQuery());
-	  return res->rowsCount();
-  }
-
-  std::uint32_t getGUIDFromDB(std::size_t mapID)
-  {
-	  auto Con(connect());
-	  if (Con == nullptr)
-		  return 0;
-	  std::unique_ptr<sql::PreparedStatement> pstmt(Con->prepareStatement("SELECT `UID` FROM `UIDs` WHERE `_map_id`=(?)"));
-	  pstmt->setInt(1, mapID);
-	  std::unique_ptr<sql::ResultSet> res(pstmt->executeQuery());
-
-	  std::uint32_t highGUID(0);
-	  if (res->rowsCount() == 0) 
-    { 
-      return 0; 
-    }
-	  while (res->next())
-	  {
-		  highGUID = res->getInt(1);
-	  }
-
-	  return highGUID;
-  }
-
-  void insertUIDinDB(std::size_t mapID, std::uint32_t NewUID)
-  {
-	  auto Con(connect());
-	  if (Con == nullptr)
-		  return;
-	  std::unique_ptr<sql::PreparedStatement> pstmt(Con->prepareStatement("INSERT INTO `UIDs` SET `_map_id`=(?), `UID`=(?)"));
-	  pstmt->setInt(1, mapID);
-	  pstmt->setInt(2, NewUID);
-	  pstmt->executeUpdate();
-  }
-
-  void updateUIDinDB (std::size_t mapID, std::uint32_t NewUID)
-  {
-	  auto Con(connect());
-	  if (Con == nullptr)
-		  return;
-	  std::unique_ptr<sql::PreparedStatement> pstmt(Con->prepareStatement("UPDATE `UIDs` SET `UID`=(?) WHERE `_map_id`=(?)"));
-	  pstmt->setInt(1, NewUID);
-	  pstmt->setInt(2, mapID);
-	  pstmt->executeUpdate();
-  }
-}
--- a/src/mysql/mysql.h
+++ b/src/mysql/mysql.h
@@ -1,15 +0,0 @@
-// This file is part of Noggit3, licensed under GNU General Public License (version 3).
-
-#pragma once
-
-#include <cinttypes>
-#include <cstddef>
-
-namespace mysql
-{
-  bool testConnection(bool report_only_err = false);
-  bool hasMaxUIDStoredDB(std::size_t mapID);
-  std::uint32_t getGUIDFromDB(std::size_t mapID);
-  void insertUIDinDB(std::size_t mapID, std::uint32_t NewUID);
-  void updateUIDinDB (std::size_t mapID, std::uint32_t NewUID);
-};
--- a/src/noggit/MapTile.cpp
+++ b/src/noggit/MapTile.cpp
@@ -1506,11 +1506,21 @@ void MapTile::setAlphaImage(QImage const& baseimage, unsigned layer, bool cleanu
      chunk->texture_set->create_temporary_alphamaps_if_needed();
      auto& temp_alphamaps = *chunk->texture_set->getTempAlphamaps();

-      for (int i = 0; i < 64; ++i)
+      float* dst = temp_alphamaps[layer].data();
+      const int base_x = k * 64;
+      const int base_y = l * 64;
+
+      for (int j = 0; j < 64; ++j)
      {
-        for (int j = 0; j < 64; ++j)
+        const int row_offset = j * 64;
+        const int img_y = base_y + j;
+
+        for (int i = 0; i < 64; ++i)
        {
-          temp_alphamaps[layer][64 * j + i] = static_cast<float>(qGray(image.pixel((k * 64) + i, (l * 64) + j)));
+          const int img_x = base_x + i;
+          QRgb px = image.pixel(img_x, img_y);
+
+          dst[row_offset + i] = static_cast<float>(qGray(px));
        }
      }

--- a/src/noggit/MapView.cpp
+++ b/src/noggit/MapView.cpp
@@ -76,11 +76,8 @@
 #include <noggit/tools/AreaTriggerTool.hpp>
 #include <noggit/StringHash.hpp>
 #include <noggit/application/NoggitApplication.hpp>
+#include <noggit/database/SqlDatabaseManager.h>

-#ifdef USE_MYSQL_UID_STORAGE
-#include <mysql/mysql.h>
-
-#endif
 #include <QtCore/QSettings>

 #include <noggit/scripting/scripting_tool.hpp>
@@ -2472,17 +2469,23 @@ void MapView::createGUI()
  set_editing_mode (editing_mode::ground);

  // do we need to do this every tick ?
-#ifdef USE_MYSQL_UID_STORAGE
  if (_settings->value("project/mysql/enabled").toBool())
  {
-      if (mysql::hasMaxUIDStoredDB(_world->getMapID()))
-      {
-        _status_database->setText("MySQL UID sync enabled: "
-            + _settings->value("project/mysql/server").toString() + ":"
-            + _settings->value("project/mysql/port").toString());
-      }
+    auto& db_mgr = Noggit::Sql::SqlDatabaseManager::instance();
+
+    if (db_mgr.testConnection(Noggit::Sql::SQLDbType::Noggit))
+    {
+       _status_database->setText("UID SQL Database is active: "
+           + _settings->value("project/mysql/server").toString() + ":"
+           + _settings->value("project/mysql/port").toString());
+    }
+    else
+    {
+      _status_database->setText("UID SQL Database is not working: "
+        + _settings->value("project/mysql/server").toString() + ":"
+        + _settings->value("project/mysql/port").toString());
+    }
  }
-#endif
 }

 void MapView::on_exit_prompt()
--- a/src/noggit/WMO.cpp
+++ b/src/noggit/WMO.cpp
@@ -1090,7 +1090,7 @@ void WMOGroup::fix_vertex_color_alpha()
    {
      r += ((r * a / 64.f) - wmo_ambient_color.x);
      g += ((g * a / 64.f) - wmo_ambient_color.y);
-      r += ((b * a / 64.f) - wmo_ambient_color.z);
+      b += ((b * a / 64.f) - wmo_ambient_color.z);
    }
    else
    {
--- a/src/noggit/database/ClientDatabase.cpp
+++ b/src/noggit/database/ClientDatabase.cpp
@@ -0,0 +1,687 @@
+#include "ClientDatabase.h"
+#include <noggit/project/CurrentProject.hpp>
+#include <noggit/application/Utils.hpp>
+#include <noggit/Log.h>
+
+#include <QString>
+#include <QSqlRecord>
+#include <QSqlField>
+#include <QElapsedTimer>
+#include <QSettings>
+
+namespace Noggit
+{
+	// namespace Sql
+		auto escapeSqlString = [](const QString& str) -> QString {
+			QString result = str;
+			result.replace("'", "''"); // double single quotes for SQL
+			return "'" + result + "'";
+			};
+
+
+		void ClientDatabase::setDatabaseMode(DatabaseMode mode)
+		{
+			_database_mode = mode;
+		}
+
+		DatabaseMode ClientDatabase::_database_mode = DatabaseMode::ClientStorage;
+		DatabaseMode ClientDatabase::databaseMode()
+		{
+			return _database_mode;
+		}
+
+		ClientDatabaseTable ClientDatabase::getTable(const std::string& tableName)
+		{
+			return ClientDatabaseTable(tableName);
+		}
+
+	bool ClientDatabaseTable::UploadDBCtoDB()
+	{
+
+		auto sql_table_name = getSqlTableName();
+
+		/*
+		if (!verifySqlTableIntegrity())
+		{
+			Log << "Table " << sql_table_name << "does not exist or has wrong structure.";
+			qDebug() << "Table " << sql_table_name.c_str() << "does not exist or has wrong structure.";
+			return false;
+		}*/
+		qDebug() << "Populating empty Table " << sql_table_name.c_str();
+
+		// insert if fresh_table, otherwise replace?
+
+		auto& row_definition = GetRecordDefinition();
+		auto sql_record_format = recordFormat();
+
+		auto client_table_iterator = getClientTable().Records();
+
+		// empty table, nothing to insert
+		if (!client_table_iterator.HasRecords())
+			return false;
+
+		QStringList column_names;
+		for (auto& sql_column_format : sql_record_format)
+		{
+			column_names.append(sql_column_format.Name.c_str());
+		}
+		int colCount = column_names.size();
+
+		auto& db_mgr = Noggit::Sql::SqlDatabaseManager::instance();
+		auto noggit_db = db_mgr.noggitDatabase();
+		QSqlQuery query(noggit_db);
+
+		// Start bulk insert ///////////////////////////////
+		QElapsedTimer timer;
+		timer.start();
+
+		const int batchSize = 2000;
+		int rowCount = 0;
+
+		noggit_db.transaction();
+
+		// query.exec("SET UNIQUE_CHECKS=0;");
+
+		QStringList rowBuffer; // holds each row as a string
+		rowBuffer.reserve(batchSize);
+
+		while (client_table_iterator.HasRecords())
+		{
+			auto& record = client_table_iterator.Next();
+			QStringList colValues;
+			colValues.reserve(column_names.size());
+
+			for (auto& column_def : row_definition.ColumnDefinitions)
+			{
+				if (column_def.Type == "int" && column_def.isID)
+				{
+					colValues.append(QString::number(record.RecordId));
+					continue;
+				}
+				auto& rowColumn = record.Columns.at(column_def.Name);
+				if (column_def.Type == "locstring")
+				{
+					for (int i = 0; i < 16; ++i)
+						colValues.append(escapeSqlString(QString::fromStdString(rowColumn.Values[i])));
+
+					auto& flagValue = record.Columns.at(column_def.Name + "_flags").Value;
+					colValues.append(QString::fromStdString(flagValue));
+				}
+				else
+				{
+					int len = (column_def.arrLength > 1) ? column_def.arrLength : 1;
+					for (int i = 0; i < len; ++i)
+					{
+						if (column_def.Type == "string")
+							colValues.append(escapeSqlString((len > 1) ? QString::fromStdString(rowColumn.Values[i])
+								: QString::fromStdString(rowColumn.Value)));
+						else // int/float
+							colValues.append((len > 1) ? QString::fromStdString(rowColumn.Values[i])
+								: QString::fromStdString(rowColumn.Value));
+					}
+				}
+			}
+
+			// append row as a single string
+			rowBuffer.append("(" + colValues.join(",") + ")");
+			rowCount++;
+
+			// flush batch
+			if (rowCount % batchSize == 0)
+			{
+				int min_size = rowCount * ((column_names.size()*2) + 2); // 2 chars minimum per column (value and comma)
+				// qDebug() << "min size" << min_size;
+				QString sql;
+				sql.reserve(min_size);
+				sql = QString("INSERT INTO `%1` (%2) VALUES ")
+					.arg(sql_table_name.c_str())
+					.arg(column_names.join(", "));
+				sql += rowBuffer.join(",");
+
+				if (!query.exec(sql))
+				{
+					qWarning() << "Batch insert failed:" << query.lastError().text();
+					// query.exec("SET UNIQUE_CHECKS=1;");
+					noggit_db.rollback();
+					return false;
+				}
+				rowBuffer.clear();
+				rowCount = 0;
+			}
+		}
+
+		// flush remaining rows
+		if (!rowBuffer.isEmpty())
+		{
+			int min_size = rowCount * ((column_names.size() * 2) + 2);
+			QString sql;
+			sql.reserve(min_size);
+			sql = QString("INSERT INTO `%1` (%2) VALUES ")
+				.arg(sql_table_name.c_str())
+				.arg(column_names.join(", "));
+			sql += rowBuffer.join(",");
+
+			if (!query.exec(sql))
+			{
+				qWarning() << "Final batch insert failed:" << query.lastError().text();
+				// query.exec("SET UNIQUE_CHECKS=1;");
+				noggit_db.rollback();
+				return false;
+			}
+		}
+		// query.exec("SET UNIQUE_CHECKS=1;");
+
+		noggit_db.commit();
+
+		// benchmark
+		qint64 elapsedMs = timer.elapsed();
+		qDebug() << "Inserted" << getClientTable().RecordCount() << "rows in" << elapsedMs << "ms ("
+			<< (getClientTable().RecordCount() * 1000.0 / elapsedMs) << " rows/sec)";
+
+		Log << "Inserted " << getClientTable().RecordCount() << " rows in " << elapsedMs << "ms ("
+			<< (getClientTable().RecordCount() * 1000.0 / elapsedMs) << " rows/sec)" << std::endl;
+
+		return true;
+	}
+
+	// executes query in client db and checks errors
+	// use isActive to check if it properly ran, not isValid.
+	QSqlQuery ClientDatabase::executeQuery(const QString& sql, bool forward_only)
+	{
+		auto db_mgr = Noggit::Sql::SqlDatabaseManager::instance().noggitDatabase();
+		QSqlQuery query(Noggit::Sql::SqlDatabaseManager::instance().noggitDatabase());
+
+		qDebug() << "Executing query : " << sql;
+		QElapsedTimer timer;
+		timer.start();
+		if (forward_only) // call this for browsing large data sets
+			query.setForwardOnly(true);
+		if (!query.exec(sql))
+		{
+			LogError << "SQL query failed:" << query.lastError().text().toStdString();
+			LogError << "Query:" << sql.toStdString();
+			// throw SqlException("Query failed: " + query.lastError().text() + "\nQuery: " + sql);
+			assert(false);
+			return QSqlQuery(); // invalid query
+		}
+
+		qint64 elapsedMs = timer.elapsed();
+		qDebug() << "Executed query in " << elapsedMs << "ms";
+
+		return query;
+	}
+
+	bool ClientDatabaseTable::createSQLTableIfNotExist()
+	{
+		auto row_definition = GetRecordDefinition();
+
+		const std::string sql_table_name = getSqlTableName();
+
+		auto db_record_format = recordFormat();
+		assert(db_record_format.size() == getClientTable().ColumnCount());
+
+		std::string statement = std::format("CREATE TABLE IF NOT EXISTS `{}` (", sql_table_name);
+
+		std::string primary_key_name;
+		for (auto& db_column_format : db_record_format)
+		{
+			statement += std::format("`{}` {}", db_column_format.Name, db_column_format.Type);
+
+			if (db_column_format.Type == "TEXT")
+			{
+				statement += " NULL"; // allow NULL by default
+			}
+			else
+			{
+				if (!db_column_format.isSigned && db_column_format.Type == "INT")
+				{
+					// assert(db_column_format.Type == "INT");
+					statement += " UNSIGNED";		// only allow int to be unsigned?
+				}
+				statement += " NOT NULL"; // allow text to be nulled
+				statement += " DEFAULT 0";
+			}
+
+			statement += ",\n";
+
+			if (db_column_format.isID)
+			{
+				assert(primary_key_name.empty()); // more than one key ? TODO
+				primary_key_name = db_column_format.Name;
+			}
+		}
+
+		if (!primary_key_name.empty())
+			statement += std::format("PRIMARY KEY (`{}`)", primary_key_name);
+
+		// Add indexes for relations
+		for (auto& db_column_format : db_record_format)
+		{
+			if (db_column_format.isRelation && !db_column_format.isID) {
+				statement += std::format(",\nINDEX (`{}`)", db_column_format.Name);
+			}
+		}
+
+		// statement += ")\n ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 DEFAULT COLLATE='utf8mb4_general_ci';";
+		statement += ")\n ENGINE = InnoDB;";
+
+		auto& db_mgr = Noggit::Sql::SqlDatabaseManager::instance();
+		bool valid_conn = db_mgr.testConnection(Noggit::Sql::SQLDbType::Noggit);
+		if (!valid_conn)
+			return false;
+		auto noggit_db = db_mgr.noggitDatabase();
+
+		QSqlQuery query(noggit_db);
+		bool success = query.exec(QString::fromStdString(statement));
+
+		if (!success)
+		{
+			qDebug() << "Failed to create table:" << query.lastError().text();
+		}
+		else
+		{
+			qDebug() << "Table " << sql_table_name.c_str() << " created.";
+
+			UploadDBCtoDB();
+		}
+
+		return success;
+	}
+
+	std::vector<DbColumnFormat> ClientDatabaseTable::recordFormat() const
+	{
+		auto record_format = std::vector<DbColumnFormat>();
+
+		auto& row_definition = GetRecordDefinition();
+		for (int col_idx = 0; col_idx < row_definition.ColumnDefinitions.size(); col_idx++)
+		{
+			auto& column_def = row_definition.ColumnDefinitions[col_idx];
+
+			bool is_locstring = false;
+
+			// convert dbd definition type names to real format
+			// TODO : map types
+			std::string sql_data_type = "INT";
+			if (BlizzardDatabaseLib::Extension::String::Compare(column_def.Type, "int"))
+			{
+				sql_data_type = "INT";
+			}
+			else if (BlizzardDatabaseLib::Extension::String::Compare(column_def.Type, "float"))
+			{
+				sql_data_type = "FLOAT";
+			}
+			else if (BlizzardDatabaseLib::Extension::String::Compare(column_def.Type, "string"))
+			{
+				sql_data_type = "TEXT";
+			}
+			else if (BlizzardDatabaseLib::Extension::String::Compare(column_def.Type, "locstring"))
+			{
+				sql_data_type = "TEXT";
+				is_locstring = true;
+			}
+			else
+				assert(false);
+
+			int array_size = 1;
+			if (column_def.arrLength > 1)
+			{
+				array_size = column_def.arrLength;
+			}
+			if (is_locstring)
+				array_size = 16;
+
+			for (int i = 0; i < array_size; i++)
+			{
+				DbColumnFormat db_col_format;
+				std::string col_name = "";
+
+				if (array_size == 1)
+				{
+					col_name = column_def.Name;
+				}
+				else if (is_locstring)
+				{
+					col_name = std::format("{}_{}", column_def.Name, dbc_string_loc_names[i]); // {MapName_lang}_{enUS}
+				}
+				else if (array_size > 1)
+				{
+					col_name = std::format("{}_{}", column_def.Name, i); // {MapName}_{0}
+				}
+				db_col_format.Name = col_name;
+				db_col_format.Type = sql_data_type;
+
+				assert(!(column_def.isID && array_size > 1));
+				db_col_format.isID = column_def.isID;
+				db_col_format.isRelation = column_def.isRelation;
+				db_col_format.isSigned = column_def.isSigned;
+
+				record_format.push_back(db_col_format);
+			}
+			if (is_locstring) // add lang mask column
+			{
+				DbColumnFormat db_col_format;
+				db_col_format.Name = std::format("{}_flags", column_def.Name);
+				db_col_format.Type = "INT";
+				db_col_format.isSigned = false;
+				db_col_format.isID = false;
+				db_col_format.isRelation = false;
+				record_format.push_back(db_col_format);
+			}
+		}
+
+		return record_format;
+	}
+
+	bool ClientDatabaseTable::verifySqlTableIntegrity()
+	{
+		auto& db_mgr = Noggit::Sql::SqlDatabaseManager::instance();
+		bool valid_conn = db_mgr.testConnection(Noggit::Sql::SQLDbType::Noggit);
+		if (!valid_conn)
+			return false;
+
+		// check if table exists
+		QString sql_table_name = getSqlTableName().c_str();
+
+		auto noggit_db = db_mgr.noggitDatabase();
+
+		// table integrity check
+		bool table_is_valid = true;
+		bool fresh_table = false;
+
+		// noggit_db.tables().contains(sql_table_name) is bugged with current qt version and mysql 8
+		QSqlQuery query_show(noggit_db);
+		if (!query_show.exec("SHOW TABLES"))
+		{
+			qWarning() << "Failed to list tables:" << query_show.lastError().text();
+			return false;
+		}
+		QStringList tables;
+		while (query_show.next())
+		{
+			tables << query_show.value(0).toString();
+		}
+
+		if (tables.contains(sql_table_name))
+		{
+			// this is also bugged...
+			// QSqlRecord sql_rec = noggit_db.record(sql_table_name);
+			// if (sql_rec.isEmpty())
+			// {
+			// 	table_is_valid = false;
+			// }
+			// else
+			// {
+			// 	// TODO verify db structure, just column count for now
+			// 	if (table.ColumnCount() != sql_rec.count())
+			// 	{
+			// 		assert(false);
+			// 		table_is_valid = false;
+			// 	}
+			// }
+		}
+		else // table doesn't exist
+		{
+			// create table
+			table_is_valid = createSQLTableIfNotExist();
+			fresh_table = true;
+		}
+
+
+		return table_is_valid;
+	}
+
+	// Structures::BlizzardDatabaseRow ClientDatabaseTable::sqlRecordToDatabaseRow(const QSqlRecord& record) const
+	Structures::BlizzardDatabaseRow ClientDatabaseTable::sqlRecordToDatabaseRow(QSqlQuery& record) const
+
+	{
+		auto& row_definition = GetRecordDefinition();
+
+		auto database_row = Structures::BlizzardDatabaseRow(-1);
+
+		int Id = -1;
+		int field_idx = 0;
+		for (int column_def_idx = 0; column_def_idx < row_definition.ColumnDefinitions.size(); ++column_def_idx)
+		{
+			auto& column_def = row_definition.ColumnDefinitions[column_def_idx];
+			auto database_column = Structures::BlizzardDatabaseColumn();
+
+			if (column_def.Type == "locstring")
+			{
+				database_column.Values.resize(16);
+				for (int loc_idx = 0; loc_idx < 16; loc_idx++)
+				{
+					database_column.Values[loc_idx] = (record.value(field_idx++).toString().toStdString());
+				}
+				// currently loc mask is set to a separate column because wdbc reader does it.
+				auto loc_mask_column = Structures::BlizzardDatabaseColumn();
+				loc_mask_column.Value = record.value(field_idx++).toString().toStdString();
+				database_row.Columns[column_def.Name + "_flags"] = loc_mask_column;
+			}
+			else // every other type than locstring
+			{
+				if (column_def.arrLength > 1) // array
+				{
+					database_column.Values.resize(column_def.arrLength);
+					for (int i = 0; i < column_def.arrLength; i++)
+					{
+						database_column.Values[i] = (record.value(field_idx++).toString().toStdString());
+					}
+				}
+				else // single value
+				{
+					database_column.Value = record.value(field_idx++).toString().toStdString();
+
+					if (column_def.isID)
+						Id = std::stoi(database_column.Value);
+				}
+			}
+			database_row.Columns[column_def.Name] = std::move(database_column);
+		}
+		assert(Id != -1); // no id found
+		database_row.RecordId = Id;
+
+		return database_row;
+	}
+
+	ClientDatabaseTable::ClientDatabaseTable(std::string tableName)
+		: _tableName(tableName), _qtTableName(tableName.c_str())
+	{
+		if (ClientDatabase::databaseMode() == DatabaseMode::Sql)
+			verifySqlTableIntegrity(); // verifySqlTableIntegrity()->createtableifnotexists()->UploadDBCtoDB()
+	};
+
+	unsigned int ClientDatabaseTable::RecordCount() const
+	{
+		unsigned int client_count = getClientTable().RecordCount();
+
+		if (ClientDatabase::databaseMode() == DatabaseMode::Sql)
+		{
+			QString sql = QString("SELECT COUNT(*) FROM `%1`").arg(getSqlTableName().c_str());
+			QSqlQuery query = ClientDatabase::executeQuery(sql);
+
+			if (query.isActive())
+			{
+				if (query.next())
+					assert(query.value(0).toUInt() == client_count);
+			}
+		}
+		return client_count;
+	}
+
+	int ClientDatabaseTable::ColumnCount() const
+	{
+		// get from parsed definition
+		int def_column_count = recordFormat().size();
+
+		int client_count = getClientTable().ColumnCount();
+		assert(def_column_count == client_count);
+
+		if (ClientDatabase::databaseMode() == DatabaseMode::Sql)
+		{
+			auto db = Noggit::Sql::SqlDatabaseManager::instance().noggitDatabase();
+			QSqlRecord rec = db.record(QString::fromStdString(getSqlTableName()));
+			int db_count = rec.count();
+			assert(db_count == def_column_count);
+		}
+
+		return def_column_count;
+	}
+
+	std::optional<Structures::BlizzardDatabaseRow> ClientDatabaseTable::RecordById(unsigned int id) const
+	{
+		auto row = Structures::BlizzardDatabaseRow(-1);
+
+		if (ClientDatabase::databaseMode() == DatabaseMode::Sql)
+			row = sqlRowById(id);
+		else
+			row = clientRowById(id);
+
+		if (row.RecordId == -1)
+			return std::nullopt;
+		else
+			return row;
+	}
+
+	Noggit::DatabaseRecordCollection ClientDatabaseTable::Records() const
+	{
+		return Noggit::DatabaseRecordCollection(*this);
+	};
+
+	/*
+	std::optional<Structures::BlizzardDatabaseRow> ClientDatabaseTable::RecordByPosition(unsigned int positionId) const
+	{
+		// TODO
+		auto row = Structures::BlizzardDatabaseRow(-1);
+		if (ClientDatabase::databaseMode() == DatabaseMode::Sql)
+		{
+			// We shouldn't do this with SQL table
+		}
+		else
+			row = getClientTable().RecordByPosition(positionId);
+
+		return std::optional<Structures::BlizzardDatabaseRow>();
+	}*/
+
+	Structures::BlizzardDatabaseRowDefinition& ClientDatabaseTable::GetRecordDefinition() const
+	{
+		return Noggit::Project::CurrentProject::get()->ClientDatabase->TableRecordDefinition(_tableName);
+	}
+
+	BlizzardDatabaseLib::BlizzardDatabaseTable& ClientDatabaseTable::getClientTable() const
+	{
+		return Noggit::Project::CurrentProject::get()->ClientDatabase->LoadTable(_tableName, readFileAsIMemStream);
+	}
+
+	// get from local dbc data memory stream in BlizzardDatabaseLib::BlizzardDatabase
+	Structures::BlizzardDatabaseRow ClientDatabaseTable::clientRowById(unsigned int id) const
+	{
+		auto record = getClientTable().RecordById(id);
+		return record;
+	}
+
+	// get from SQL request to noggit db
+	// never use this function for more than 1 rows, implement a new bulk function
+	Structures::BlizzardDatabaseRow ClientDatabaseTable::sqlRowById(unsigned int id) const
+	{
+		QString sql_table_name = getSqlTableName().c_str();
+		QString sql = QString("SELECT * FROM `%1` WHERE ID = %2").arg(sql_table_name).arg(id);
+
+		auto query = ClientDatabase::executeQuery(sql);
+
+		if (!query.isActive())
+			return BlizzardDatabaseLib::Structures::BlizzardDatabaseRow(-1);
+
+		// auto row_definition = GetRecordDefinition();
+
+		if (query.next())
+		{
+			// QSqlRecord record = query.record(); // slow af
+			auto database_row = sqlRecordToDatabaseRow(query);
+
+			return database_row;
+		}
+		else
+		{
+			LogError << "SQL : No row found in" << sql_table_name.toStdString() << "for ID =" << id;
+			qWarning() << "SQL : No row found in" << sql_table_name << "for ID =" << id;
+			return BlizzardDatabaseLib::Structures::BlizzardDatabaseRow();
+		}
+
+	}
+	
+	const std::string ClientDatabaseTable::getSqlTableName(unsigned int build_id) const
+	{
+		if (build_id == 0)
+			build_id = Noggit::Project::CurrentProject::get()->buildId();
+
+		std::string table = std::format("db_{}_{}", _tableName, build_id);
+
+		// convert to lowercase for compatibility with SQL
+		std::transform(table.begin(), table.end(), table.begin(),
+			[](unsigned char c) { return std::tolower(c); });
+
+		return table;
+	}
+
+	DatabaseRecordCollection::DatabaseRecordCollection(const ClientDatabaseTable& table)
+		:_table(table), /*_mode(mode),*/ _client_iterator(_table.getClientTable().Records())
+	{
+		if (ClientDatabase::databaseMode() == DatabaseMode::Sql)
+		{
+			QString sql = QString("SELECT * FROM `%1`").arg(_table.getSqlTableName().c_str()); //  ORDER BY ID ?
+
+			_query = ClientDatabase::executeQuery(sql, true);
+			_querry_valid = _query.isActive(); // if query.exec ran properly
+
+			querryAdvance();
+		}
+	}
+
+	bool DatabaseRecordCollection::HasRecords()
+	{
+		if (ClientDatabase::databaseMode() == DatabaseMode::ClientStorage)
+			return _client_iterator.HasRecords();
+		else
+		{
+			return _querry_valid && _hasNext;
+		}
+	}
+
+	Structures::BlizzardDatabaseRow DatabaseRecordCollection::Next()
+	{
+		if (ClientDatabase::databaseMode() == DatabaseMode::ClientStorage)
+			return _client_iterator.Next();
+		else
+		{
+			if (!_querry_valid || !_hasNext)
+			{
+				assert(false);
+				return Structures::BlizzardDatabaseRow(); // empty
+			}
+
+			// always store one row in advance to know if it's the last one
+			// auto row = _table.sqlRecordToDatabaseRow(_nextRecord);
+			assert(_nextRecord.RecordId != -1);
+
+			querryAdvance();
+
+			return _nextRecord;
+		}
+
+	}
+
+	void DatabaseRecordCollection::querryAdvance()
+	{
+		if (_query.next())
+		{
+			// _nextRecord = _query.record();
+			_nextRecord = _table.sqlRecordToDatabaseRow(_query);
+			_hasNext = true;
+		}
+		else
+		{
+			_hasNext = false;
+		}
+	}
+
+}
--- a/src/noggit/database/ClientDatabase.h
+++ b/src/noggit/database/ClientDatabase.h
@@ -0,0 +1,148 @@
+#pragma once
+#include <blizzard-database-library/include/BlizzardDatabase.h>
+#include <blizzard-database-library/include/structures/FileStructures.h>
+
+#include <noggit/database/SqlDatabaseManager.h>
+
+#include <optional>
+
+#include <QString>
+#include <QSqlRecord>
+
+
+constexpr const char* dbc_string_loc_names[16] = { "enUS", "koKR", "frFR", "deDE", "zhCN",
+                              "zhTW", "esES", "esMX", "ruRU", "jaJP", "ptPT", "itIT",
+                              "unk_12", "unk_13", "unk_14", "unk_15" };
+
+using namespace BlizzardDatabaseLib;
+namespace Noggit
+{
+  struct DbColumnFormat
+  {
+    std::string Type = "";
+    std::string Name = "";
+    // int size;
+    bool isID = false;
+    bool isRelation = false;
+    bool isSigned = true;
+  };
+
+  enum class DatabaseMode
+  {
+    Sql,
+    ClientStorage
+  };
+
+  struct SqlException : public std::runtime_error
+  {
+    SqlException(const QString& msg)
+      : std::runtime_error(msg.toStdString()) {}
+  };
+
+  class Noggit::ClientDatabaseTable;
+  // calls client or server db adaptively. so /sql/ is not really a good location
+  class ClientDatabase
+  {
+    friend class ClientDatabaseTable;
+  public:
+
+    static void setDatabaseMode(DatabaseMode mode);
+    static DatabaseMode databaseMode(); // sql or client storage
+
+    static ClientDatabaseTable getTable(const std::string& tableName);
+
+    static void saveTable(const std::string& tableName);
+  
+    // static std::optional<Structures::BlizzardDatabaseRow> getRowById(const std::string& tableName, unsigned int id); // constructs a row either from db or client
+  
+    static void TODODeploySqlToClient();
+
+    static QSqlQuery executeQuery(const QString& sql, bool forward_only = true); // forward onl means can't browse query backward, but massively speeds up forward iteration
+  
+  private:
+    static DatabaseMode _database_mode;
+  
+    static Structures::BlizzardDatabaseRow clientRowById(const std::string& tableName, unsigned int id);
+    static Structures::BlizzardDatabaseRow sqlRowById(const std::string& tableName, unsigned int id);
+  
+
+  };
+
+  // TODO can subclass BlizzardDatabaseRecordCollection instead
+  class DatabaseRecordCollection
+  {
+  public:
+    DatabaseRecordCollection(const ClientDatabaseTable& table);
+    bool HasRecords();
+    Structures::BlizzardDatabaseRow Next();
+    // Structures::BlizzardDatabaseRow First();
+    // Structures::BlizzardDatabaseRow Last();
+
+  private:
+    const ClientDatabaseTable& _table;
+
+    // client
+    BlizzardDatabaseRecordCollection _client_iterator;
+
+    // sql stuff
+    QSqlQuery _query;
+    bool _querryHasStarted = false;
+    bool _querry_valid = false;
+    bool _hasNext = false;
+    // QSqlRecord _nextRecord;
+    Structures::BlizzardDatabaseRow _nextRecord;
+    void querryAdvance();
+  };
+
+  // interface table that gets data either from sql or raw dbc
+  // TODO : can just make BlizzardDatabaseTable subclass this.
+  class ClientDatabaseTable
+  {
+    friend class DatabaseRecordCollection;
+
+  private:
+    const std::string _tableName;
+    const QString _qtTableName;
+    const Structures::BlizzardDatabaseRowDefinition _row_definition;
+
+  public:
+    ClientDatabaseTable(std::string tableName);
+
+    // table info
+    const std::string Name() const { return _tableName; };
+    unsigned int RecordCount() const;
+    int ColumnCount() const;
+    int getRecordSize() const;
+    Structures::BlizzardDatabaseRowDefinition& GetRecordDefinition() const;
+
+    // get rows data
+    std::optional<Structures::BlizzardDatabaseRow> RecordById(unsigned int id) const;
+    // std::optional<Structures::BlizzardDatabaseRow> RecordByPosition(unsigned int positionId) const;
+    // CheckIfIdExists
+    Noggit::DatabaseRecordCollection Records() const;// { return Noggit::DatabaseRecordCollection(*this); };
+
+    // modify data
+    // Record addRecord(size_t id, size_t id_field = 0);
+    // Record addRecordCopy(size_t id, size_t id_from, size_t id_field = 0);
+    // void removeRecord(size_t id, size_t id_field = 0);
+    // int getEmptyRecordID(size_t id_field = 0);
+
+
+  private:
+    // for internal use only, get the client storage
+    BlizzardDatabaseLib::BlizzardDatabaseTable&  getClientTable() const;
+
+    Structures::BlizzardDatabaseRow clientRowById(unsigned int id) const;
+    Structures::BlizzardDatabaseRow sqlRowById(unsigned int id) const;
+
+    // sql helpers
+    bool UploadDBCtoDB();
+    const std::string getSqlTableName(unsigned int build_id = 0) const; // get automatically from project if default(0)
+    std::vector<DbColumnFormat> recordFormat() const; // true record format for all columns, not array size/loc etc. eg returns all 17 columns for loc.
+    bool createSQLTableIfNotExist();
+    bool verifySqlTableIntegrity();
+    // Structures::BlizzardDatabaseRow sqlRecordToDatabaseRow(const QSqlRecord& record) const;
+    Structures::BlizzardDatabaseRow sqlRecordToDatabaseRow(QSqlQuery& query) const;
+  };
+}
+
--- a/src/noggit/database/SqlDatabaseManager.cpp
+++ b/src/noggit/database/SqlDatabaseManager.cpp
--- a/src/noggit/database/SqlDatabaseManager.h
+++ b/src/noggit/database/SqlDatabaseManager.h
@@ -0,0 +1,236 @@
+#pragma once
+#include <QSqlDatabase>
+#include <QString>
+#include <QSqlError>
+#include <QDebug>
+#include <QSqlQuery>
+#include <QMap>
+#include <QStringList>
+#include <QMessageBox>
+
+// Connections are currently not thread safe ! would need temporary connection copies per thread.
+namespace Noggit::Sql
+{
+  enum class SQLDbType
+  {
+    Noggit,
+    World
+  };
+  
+  enum class SqlDriver
+  {
+    MySQL,
+    // SQLite,
+    // PostgreSQL
+  };
+
+  struct DbConfig
+  {
+    QString host;
+    int port = 3306;
+    QString dbName;
+    QString user;
+    QString password;
+  };
+
+  class SqlDatabaseManager
+  {
+  public:
+    static SqlDatabaseManager& instance()
+    {
+      static SqlDatabaseManager _instance;
+      return _instance;
+    }
+  
+    bool initializeDbConnection(SQLDbType type,
+      const QString& host, int port,
+      const QString& dbName, const QString& user, const QString& password)
+    {
+      DbConfig newConfig{ host, port, dbName, user, password };
+
+      // If config is unchanged and connection is alive, nothing to do
+      if (configs[type].host == host &&
+        configs[type].port == port &&
+        configs[type].dbName == dbName &&
+        configs[type].user == user &&
+        configs[type].password == password &&
+        testConnection(type))
+      {
+        return true;
+      }
+      configs[type] = newConfig;
+
+      QString connName = connectionName(type);
+
+      // remove old connection if it exists
+      if (QSqlDatabase::contains(connName))
+      {
+        QSqlDatabase oldDb = QSqlDatabase::database(connName);
+        if (oldDb.isOpen())
+          oldDb.close();
+        QSqlDatabase::removeDatabase(connName);
+      }
+
+      // Create new connection
+      QSqlDatabase db = QSqlDatabase::addDatabase(driverToString(SqlDriver::MySQL), connName);
+      db.setHostName(host);
+      db.setPort(port);
+      // db.setDatabaseName(dbName);
+      db.setUserName(user);
+      db.setPassword(password);
+
+      // connect to my sql without a database
+      if (!db.open())
+      {
+        qWarning() << "Failed to connect to database " << dbName << "on host" << host << ":" << db.lastError().text();
+        return false;
+      }
+
+      // Create database if it doesn't exist
+      QSqlQuery query(db);
+      if (!query.exec(QStringLiteral("CREATE DATABASE IF NOT EXISTS `%1`").arg(dbName)))
+      {
+        qWarning() << "Failed to open database " << dbName << ":" << db.lastError().text();
+        return false;
+      }
+
+      // now attempt to connect to db
+      // Need to close connection first because it needs to create the connection with a databasename.
+      db.close();
+      db.setDatabaseName(dbName);
+      if (!db.open())
+      {
+        qDebug() << "Failed to open database:" << db.lastError().text();
+        return false;
+      }
+      qDebug() << db.tables();
+
+      return true;
+    }
+  
+    // requires database to be initialized
+    bool testConnection(SQLDbType type) const
+    {
+      if (!configs.contains(type))
+      {
+        qWarning() << "No configuration for database type" << static_cast<int>(type);
+      }
+
+      QSqlDatabase db = databaseConnection(type);
+      if (db.isValid() && db.isOpen())
+        return true;
+
+      // If the connection is invalid or closed, attempt to open it
+      if (!db.open())
+      {
+        qWarning() << "Connection test failed for" << db.connectionName() << ":" << db.lastError().text();
+        qDebug() << db.lastError().nativeErrorCode();
+        qDebug() << db.lastError().databaseText();
+        return false;
+      }
+      else
+      {
+        qDebug() << "Reconnected to database" << db.connectionName();
+      }
+
+      return true;
+    }
+  
+    bool testConnectionWithPopup(SQLDbType type, bool report_only_error) const
+    {
+      QMessageBox prompt;
+      prompt.setWindowFlag(Qt::WindowStaysOnTopHint);
+      bool success = false;
+      std::stringstream promptText;
+      try
+      {
+        success = testConnection(type);
+      }
+      catch (std::exception& e)
+      {
+        success = false;
+
+        promptText << "\n# ERR: " << e.what();
+      }
+
+      if (success && !report_only_error)
+      {
+        prompt.setIcon(QMessageBox::Information);
+        prompt.setText("Succesfully connected to MySQL database.");
+        prompt.setWindowTitle("Success");
+
+        if (!report_only_error)
+          prompt.exec();
+      }
+      else if (!success)
+      {
+        prompt.setIcon(QMessageBox::Warning);
+        prompt.setText("Failed to load MySQL database, check your settings. \nIf you did not intend to use this feature, disable it in Noggit->settings->MySQL");
+        prompt.setWindowTitle("Noggit Database Error");
+        // disable if connection is not valid
+        // settings.value("project/mysql/enabled") = false;
+
+        promptText << databaseConnection(type).lastError().text().toStdString() << std::endl;
+        promptText << databaseConnection(type).lastError().nativeErrorCode().toStdString() << std::endl;
+        promptText << databaseConnection(type).lastError().databaseText().toStdString() << std::endl;
+
+        prompt.setInformativeText(promptText.str().c_str());
+        prompt.exec();
+      }
+
+      return success;
+    }
+
+    bool isDriverAvailable()
+    {
+      QStringList availableDrivers = QSqlDatabase::drivers();
+      return availableDrivers.contains(driverToString(_sql_driver), Qt::CaseInsensitive);
+    }
+
+    QSqlDatabase databaseConnection(SQLDbType type) const
+    {
+      return QSqlDatabase::database(connectionName(type));
+    }
+  
+    QSqlDatabase noggitDatabase() const
+    {
+      return SqlDatabaseManager::databaseConnection(SQLDbType::Noggit);
+    }
+  
+    QSqlDatabase worldDatabase() const
+    {
+      return SqlDatabaseManager::databaseConnection(SQLDbType::World);
+    }
+  
+  private:
+    SqlDatabaseManager() = default;
+    ~SqlDatabaseManager() = default;
+    SqlDatabaseManager(const SqlDatabaseManager&) = delete;
+    SqlDatabaseManager& operator=(const SqlDatabaseManager&) = delete;
+
+    SqlDriver _sql_driver = SqlDriver::MySQL;
+
+    mutable QMap<SQLDbType, DbConfig> configs; // stores current configuration per DB type
+  
+    QString connectionName(SQLDbType type) const
+    {
+      switch (type)
+      {
+      case SQLDbType::Noggit: return "noggit";
+      case SQLDbType::World: return "world";
+      }
+      return {};
+    }
+  
+    QString driverToString(SqlDriver driver)
+    {
+      switch (driver)
+      {
+        case SqlDriver::MySQL: return "QMYSQL";
+        // case SqlDriver::SQLite: return "QSQLITE";
+        // case SqlDriver::PostgreSQL: return "QPSQL";
+      }
+      return "";
+    }
+  };
+}
--- a/src/noggit/database/SqlUIDStorage.cpp
+++ b/src/noggit/database/SqlUIDStorage.cpp
@@ -0,0 +1,107 @@
+#include "SqlUIDStorage.h"
+
+bool Noggit::Sql::SqlUIDStorage::hasMaxUIDStoredDB(std::size_t mapID)
+{
+	auto& db_mgr = SqlDatabaseManager::instance();
+
+	bool valid_conn = db_mgr.testConnection(SQLDbType::Noggit);
+	if (!valid_conn)
+		return false;
+
+	auto noggit_db = db_mgr.noggitDatabase();
+
+	QSqlQuery query(noggit_db);
+	query.prepare(QStringLiteral("SELECT * FROM `UIDs` WHERE `_map_id` = ?"));
+	query.addBindValue(static_cast<int>(mapID));
+
+	if (!query.exec())
+	{
+		qWarning() << "Failed to check UIDs:" << query.lastError().text();
+		return false;
+	}
+
+	return query.next(); // true if at least one row exists
+}
+
+std::uint32_t Noggit::Sql::SqlUIDStorage::getGUIDFromDB(std::size_t mapID)
+{
+	auto& db_mgr = Sql::SqlDatabaseManager::instance();
+
+	bool valid_conn = db_mgr.testConnection(SQLDbType::Noggit);
+	if (!valid_conn)
+		return 0;
+
+	QSqlDatabase noggit_db = db_mgr.noggitDatabase();
+	/////
+	QSqlQuery query(noggit_db);
+	query.prepare(QStringLiteral("SELECT `UID` FROM `UIDs` WHERE `_map_id` = ?"));
+	query.addBindValue(static_cast<int>(mapID));
+
+	if (!query.exec())
+	{
+		qWarning() << "Failed to fetch UID from db:" << query.lastError().text();
+		return 0;
+	}
+
+	if (query.next())
+		return query.value(0).toUInt();
+
+	return 0; // no rows
+
+	// Optional, find highest GUID of all maps.
+	// "SELECT `UID` FROM `UIDs`"
+	// std::uint32_t highGUID(0);
+	// if (res->rowsCount() == 0)
+	// {
+	// 	return 0;
+	// }
+	// while (res->next())
+	// {
+	// 	highGUID = res->getInt(1);
+	// }
+	// 
+	// 
+	// return highGUID;
+}
+
+void Noggit::Sql::SqlUIDStorage::insertUIDinDB(std::size_t mapID, std::uint32_t NewUID)
+{
+	auto& db_mgr = Sql::SqlDatabaseManager::instance();
+	bool valid_conn = db_mgr.testConnection(SQLDbType::Noggit);
+	if (!valid_conn)
+	{
+			return;
+	}
+
+	QSqlDatabase noggit_db = db_mgr.noggitDatabase();
+
+	QSqlQuery query(noggit_db);
+	query.prepare(QStringLiteral("INSERT INTO `UIDs` (`_map_id`, `UID`) VALUES (?, ?)"));
+	query.addBindValue(static_cast<int>(mapID));
+	query.addBindValue(static_cast<int>(NewUID));
+
+	if (!query.exec())
+	{
+		qWarning() << "Failed to insert UID in db: " << query.lastError().text();
+	}
+}
+
+void Noggit::Sql::SqlUIDStorage::updateUIDinDB(std::size_t mapID, std::uint32_t NewUID)
+{
+	auto& db_mgr = Sql::SqlDatabaseManager::instance();
+
+	if (!db_mgr.testConnection(SQLDbType::Noggit))
+		return;
+
+	QSqlDatabase noggit_db = db_mgr.noggitDatabase();
+
+	QSqlQuery query(noggit_db);
+	query.prepare(QStringLiteral("UPDATE `UIDs` SET `UID` = ? WHERE `_map_id` = ?"));
+	query.addBindValue(static_cast<int>(NewUID));
+	query.addBindValue(static_cast<int>(mapID));
+
+	if (!query.exec())
+	{
+		qWarning() << "Failed to update UID:" << query.lastError().text();
+	}
+}
--- a/src/noggit/database/SqlUIDStorage.h
+++ b/src/noggit/database/SqlUIDStorage.h
@@ -0,0 +1,16 @@
+#pragma once
+#include <noggit/database/SqlDatabaseManager.h>
+
+namespace Noggit::Sql
+{
+  class SqlUIDStorage
+  {
+  public:
+
+    static bool hasMaxUIDStoredDB(std::size_t mapID);
+    static std::uint32_t getGUIDFromDB(std::size_t mapID);
+    static void insertUIDinDB(std::size_t mapID, std::uint32_t NewUID);
+    static void updateUIDinDB(std::size_t mapID, std::uint32_t NewUID);
+  };
+}
+
--- a/src/noggit/database/clientDbDefinitions/MapDB.h
+++ b/src/noggit/database/clientDbDefinitions/MapDB.h
@@ -0,0 +1,44 @@
+#pragma once
+#include <noggit/database/ClientDatabase.h>
+
+// experimental
+
+
+struct MapDbRow : BlizzardDatabaseLib::Structures::BlizzardDatabaseRow
+{
+  uint AreaType() { return getUInt("InstanceType"); };        // uint
+};
+
+class MapDb : public Noggit::ClientDatabaseTable
+{
+public:
+  MapDb() :
+    ClientDatabaseTable("Map")
+  {}
+    uint MapID() ;        // uint
+
+    std::optional<MapDbRow> mapRecordById(unsigned int id) { return static_cast<MapDbRow>(RecordById(id).value()); };
+
+  /// Fields
+  static const size_t MapID = 0;        // uint
+  static const size_t InternalName = 1;    // string
+  static const size_t AreaType = 2;      // uint
+  static const size_t Flags = 3;      // uint
+  static const size_t IsBattleground = 4;    // uint 
+  static const size_t Name = 5;        // loc
+  static const size_t AreaTableID = 22;    // uint 
+  static const size_t MapDescriptionAlliance = 23;    // loc 
+  static const size_t MapDescriptionHorde = 40;    // loc 
+  static const size_t LoadingScreen = 57;    // uint [LoadingScreen]
+  static const size_t minimapIconScale = 58;    // uint [LoadingScreen]
+  static const size_t corpseMapID = 59; //	iRefID	Points to column 1, -1 if none
+  static const size_t corpseX = 60; //	Float	The X - Coord of the instance entrance
+  static const size_t corpseY = 61; //	Float	The Y - Coord of the instance entrance
+  static const size_t TimeOfDayOverride = 62; //	Integer	Set to - 1 for everything but Orgrimmar and Dalaran arena.For those, the time of day will change to this.
+  static const size_t ExpansionID = 63; //	Integer	Vanilla : 0, BC : 1, WotLK : 2
+  static const size_t RaidOffset = 64; //	Integer
+  static const size_t NumberOfPlayers = 65; //	Integer	Used for reset time?
+
+  static std::string getMapName(int pMapID);
+  static int findMapName(const std::string& map_name);
+};
--- a/src/noggit/map_index.cpp
+++ b/src/noggit/map_index.cpp
@@ -9,13 +9,11 @@
 #include <noggit/ActionManager.hpp>
 #include <noggit/Action.hpp>
 #include <noggit/project/CurrentProject.hpp>
-#ifdef USE_MYSQL_UID_STORAGE
-  #include <mysql/mysql.h>
-#endif
 #include <noggit/map_index.hpp>
 #include <noggit/uid_storage.hpp>
 #include <noggit/application/NoggitApplication.hpp>
 #include <ClientFile.hpp>
+#include <noggit/database/SqlUIDStorage.h>

 #include <QtCore/QSettings>
 #include <QByteArray>
@@ -776,14 +774,13 @@ uint32_t MapIndex::newGUID()
 {
  std::unique_lock<std::mutex> lock (_mutex);

-#ifdef USE_MYSQL_UID_STORAGE
  QSettings settings;

  if (settings.value ("project/mysql/enabled", false).toBool())
  {
-    mysql::updateUIDinDB(_map_id, highestGUID + 1); // update the highest uid in db, note that if the user don't save these uid won't be used (not really a problem tho) 
+    // update the highest uid in db, note that if the user don't save these uid won't be used (not really a problem tho) 
+    Noggit::Sql::SqlUIDStorage::updateUIDinDB(_map_id, highestGUID + 1);
  }
-#endif
  return ++highestGUID;
 }

@@ -1135,21 +1132,19 @@ void MapIndex::searchMaxUID()

 void MapIndex::saveMaxUID()
 {
-#ifdef USE_MYSQL_UID_STORAGE
  QSettings settings;

  if (settings.value ("project/mysql/enabled", false).toBool())
  {
-    if (mysql::hasMaxUIDStoredDB(_map_id))
+    if (Noggit::Sql::SqlUIDStorage::hasMaxUIDStoredDB(_map_id))
    {
-	    mysql::updateUIDinDB(_map_id, highestGUID);
+      Noggit::Sql::SqlUIDStorage::updateUIDinDB(_map_id, highestGUID);
    }
    else
    {
-	    mysql::insertUIDinDB(_map_id, highestGUID);
+      Noggit::Sql::SqlUIDStorage::insertUIDinDB(_map_id, highestGUID);
    }
  }
-#endif
  // save the max UID on the disk (always save to sync with the db if used
  uid_storage::saveMaxUID (_map_id, highestGUID);
 }
@@ -1157,16 +1152,14 @@ void MapIndex::saveMaxUID()
 void MapIndex::loadMaxUID()
 {
  highestGUID = uid_storage::getMaxUID (_map_id);
-#ifdef USE_MYSQL_UID_STORAGE
  QSettings settings;

  if (settings.value ("project/mysql/enabled", false).toBool())
  {
-    highestGUID = std::max(mysql::getGUIDFromDB(_map_id), highestGUID);
+    highestGUID = std::max(Noggit::Sql::SqlUIDStorage::getGUIDFromDB(_map_id), highestGUID);
    // save to make sure the db and disk uid are synced
    saveMaxUID();
  }
-#endif
 }

 void MapIndex::loadMinimapMD5translate()
--- a/src/noggit/project/ApplicationProject.cpp
+++ b/src/noggit/project/ApplicationProject.cpp
@@ -259,6 +259,11 @@ namespace Noggit::Project
      _projectWriter = std::make_shared<ApplicationProjectWriter>();
    }

+    unsigned int NoggitProject::buildId()
+    {
+      return ClientDatabase->getBuild();
+    }
+
    void NoggitProject::createBookmark(const NoggitProjectBookmarkMap& bookmark)
    {
      Bookmarks.push_back(bookmark);
--- a/src/noggit/project/ApplicationProject.h
+++ b/src/noggit/project/ApplicationProject.h
@@ -117,6 +117,8 @@ namespace Noggit::Project
    NoggitExtraMapData ExtraMapData;
    NoggitProject();

+    unsigned int buildId();
+
    void createBookmark(const NoggitProjectBookmarkMap& bookmark);

    void deleteBookmark();
--- a/src/noggit/texture_set.cpp
+++ b/src/noggit/texture_set.cpp
@@ -1528,23 +1528,34 @@ void TextureSet::updateDoodadMapping()
    if (debug_test)
        blizzard_mapping_readable = getDoodadMappingReadable();

-    // 8x8 bits per unit
-    for (int unit_x = 0; unit_x < 8; unit_x++)
+    constexpr int TILE_SIZE = 64;
+    constexpr int UNIT_SIZE = 8;
+    constexpr int NUM_UNITS = TILE_SIZE / UNIT_SIZE; // 8
+
+    for (int unit_x = 0; unit_x < NUM_UNITS; unit_x++)
    {
-        for (int unit_y = 0; unit_y < 8; unit_y++)
+        for (int unit_y = 0; unit_y < NUM_UNITS; unit_y++)
        {
            int layer_totals[4]{ 0,0,0,0 };

+            const int unit_base_y = unit_y * UNIT_SIZE;
+            const int unit_base_x = unit_x * UNIT_SIZE;
+
            // 8x8 bits per unit
-            for (int x = 0; x < 8; x++)
+            for (int y = 0; y < UNIT_SIZE; y++)
            {
-                for (int y = 0; y < 8; y++)
+                const int row_base = (unit_base_y + y) * TILE_SIZE + unit_base_x;
+
+                for (int x = 0; x < UNIT_SIZE; x++)
                {
                    int base_alpha = 255;

+                    const int alpha_pos = row_base + x;
+                    // const int alpha_pos = (unit_y * 8 + y) * 64 + (unit_x * 8 + x);
+
                    for (int alpha_layer = 0; alpha_layer < (nTextures - 1); ++alpha_layer)
                    {
-                        int alpha = static_cast<int>(alphamaps[alpha_layer]->getAlpha((unit_y * 8 + y) * 64 + (unit_x * 8 + x)));
+                        const int alpha = static_cast<int>(alphamaps[alpha_layer]->getAlpha(alpha_pos));

                        layer_totals[alpha_layer+1] += alpha;

@@ -1637,7 +1648,12 @@ namespace
 {
   std::uint8_t float_alpha_to_uint8(float a)
  {
-    return static_cast<std::uint8_t>(std::max(0.f, std::min(255.f, std::round(a))));
+    // return static_cast<std::uint8_t>(std::max(0.f, std::min(255.f, std::round(a))));
+
+     int v = static_cast<int>(a + 0.5f);
+     if (static_cast<unsigned>(v) > 255u)
+       v = v < 0 ? 0 : 255;
+     return static_cast<std::uint8_t>(v);
  }
 }

@@ -1648,25 +1664,29 @@ bool TextureSet::apply_alpha_changes()
    tmp_edit_values.reset();
    return false;
  }
+  constexpr int ALPHA_SIZE = 64 * 64;

  auto& new_amaps = *tmp_edit_values;
-  std::array<std::uint16_t, 64 * 64> totals;
+  std::array<std::uint16_t, ALPHA_SIZE> totals;
  totals.fill(0);

  for (int alpha_layer = 0; alpha_layer < nTextures - 1; ++alpha_layer)
  {
-    std::array<std::uint8_t, 64 * 64> values;
+    std::array<std::uint8_t, ALPHA_SIZE> values;

-    for (int i = 0; i < 64 * 64; ++i)
+    auto& tmp_layer = new_amaps[alpha_layer + 1];
+
+    for (int i = 0; i < ALPHA_SIZE; ++i)
    {
-      values[i] = float_alpha_to_uint8(new_amaps[alpha_layer + 1][i]);
-      totals[i] += values[i];
+      uint8_t new_value = float_alpha_to_uint8(tmp_layer[i]);
+      values[i] = new_value;
+      uint16_t total = totals[i] += new_value;

      // remove the possible overflow with rounding
      // max 2 if all 4 values round up so it won't change the layer's alpha much
-      if (totals[i] > 255)
+      if (total > 255)
      {
-        values[i] -= static_cast<std::uint8_t>(totals[i] - 255);
+        new_value -= static_cast<std::uint8_t>(total - 255);
      }
    }

--- a/src/noggit/ui/tools/MapCreationWizard/Ui/MapCreationWizard.cpp
+++ b/src/noggit/ui/tools/MapCreationWizard/Ui/MapCreationWizard.cpp
@@ -14,6 +14,7 @@
 #include <noggit/World.h>

 #include <blizzard-database-library/include/BlizzardDatabase.h>
+#include <noggit/database/ClientDatabase.h>

 #include <QApplication>
 #include <QButtonGroup>
@@ -33,6 +34,8 @@
 #include <QSpinBox>
 #include <QStackedWidget>
 #include <QWheelEvent>
+#include <QtCore/QSettings>
+#include <QElapsedTimer>

 #include <filesystem>

@@ -80,10 +83,47 @@ MapCreationWizard::MapCreationWizard(std::shared_ptr<Project::NoggitProject> pro
  _corpse_map_id->addItem("None");
  _corpse_map_id->setItemData(0, QVariant (-1));

-  // Fill selector combo

+  // TEST BENCHMARK SQL STUFF/////////////////////////
+  /*
+  {
+    Log << "Iterating table : " << "WMOAreaTable" << std::endl;
+    QElapsedTimer timer;
+    timer.start();
+
+    auto testtable = ClientDatabase::getTable("WMOAreaTable");
+    // auto& testtable = Noggit::Project::CurrentProject::get()->ClientDatabase->LoadTable("WMOAreaTable", readFileAsIMemStream);
+    // 
+    // for (DBCFile::Iterator i = gWMOAreaTableDB.begin(); i != gWMOAreaTableDB.end(); ++i)
+    // {
+    //   unsigned int lol = i->getUInt(WMOAreaTableDB::ID);
+    //   for (int ii = 0; ii < WMOAreaTableDB::Name; ii++)
+    //   {
+    //     unsigned int lol = i->getUInt(ii);
+    //   }
+    //   i->getLocalizedString(WMOAreaTableDB::Name);
+    // }
+
+    qint64 elapsedMs = timer.elapsed();
+    Log << "gettable() in : " << elapsedMs << "ms" << std::endl;
+
+    auto iterator = testtable.Records();
+    while (iterator.HasRecords())
+    {
+      auto record = iterator.Next();
+    }
+    elapsedMs = timer.elapsed();
+    Log << "fully iterated table in  : " << elapsedMs << "ms" << std::endl;
+
+  }*/
+  /////////////////////////////////////////////////////
+
+
+  // Fill selector combo
  const auto& table = std::string("Map");
-  auto mapTable = _project->ClientDatabase->LoadTable(table, readFileAsIMemStream);
+
+  auto mapTable = ClientDatabase::getTable(table);
+  // auto& mapTable = Noggit::Project::CurrentProject::get()->ClientDatabase->LoadTable(table, readFileAsIMemStream);

  int count = 0;
  auto iterator = mapTable.Records();
@@ -104,8 +144,6 @@ MapCreationWizard::MapCreationWizard(std::shared_ptr<Project::NoggitProject> pro
      count++;
  }

-  _project->ClientDatabase->UnloadTable("Map");
-
  auto add_btn = new QPushButton("New",this);
  add_btn->setIcon(Noggit::Ui::FontAwesomeIcon(Noggit::Ui::FontAwesome::plus));
  layout_selector->addWidget(add_btn);
@@ -628,8 +666,11 @@ void MapCreationWizard::selectMap(int map_id)

  // int map_id = world->getMapID();

-  auto table = _project->ClientDatabase->LoadTable("Map", readFileAsIMemStream);
-  auto record = table.Record(map_id);
+  auto table = ClientDatabase::getTable("Map");
+  auto rec_opt = table.RecordById(map_id);
+  if (!rec_opt)
+    return;
+  auto& record = *rec_opt;

  _cur_map_id = map_id;

@@ -722,9 +763,7 @@ void MapCreationWizard::selectMap(int map_id)

  _max_players->setValue(std::atoi(maxPlayers.c_str()));

-  _project->ClientDatabase->UnloadTable("Map");
-
-  auto difficulty_table = _project->ClientDatabase->LoadTable("MapDifficulty", readFileAsIMemStream);
+  auto difficulty_table = ClientDatabase::getTable("MapDifficulty");

  auto iterator = difficulty_table.Records();

@@ -745,7 +784,6 @@ void MapCreationWizard::selectMap(int map_id)
          _difficulty_type->insertItem(difficulty_type, diff_text.c_str(), QVariant(record_id));
      }
  }
-  _project->ClientDatabase->UnloadTable("MapDifficulty");
  _difficulty_type->setCurrentIndex(0);
  selectMapDifficulty();

@@ -761,8 +799,11 @@ void MapCreationWizard::selectMapDifficulty()
    if (!selected_difficulty_id)
        return;

-    auto difficulty_table = _project->ClientDatabase->LoadTable("MapDifficulty", readFileAsIMemStream);
-    auto record = difficulty_table.Record(selected_difficulty_id);
+    auto difficulty_table = ClientDatabase::getTable("MapDifficulty");
+    auto rec_opt = difficulty_table.RecordById(selected_difficulty_id);
+    if (!rec_opt)
+      return;
+    auto& record = *rec_opt;

    //_difficulty_type;
    _difficulty_req_message->fill(record, "Message_lang");
@@ -772,8 +813,6 @@ void MapCreationWizard::selectMapDifficulty()

    _difficulty_max_players->setValue(std::atoi(record.Columns["MaxPlayers"].Value.c_str()));
    _difficulty_string->setText(record.Columns["Difficultystring"].Value.c_str());
-
-    _project->ClientDatabase->UnloadTable("MapDifficulty");
 }

 void MapCreationWizard::wheelEvent(QWheelEvent* event)
--- a/src/noggit/ui/tools/PresetEditor/Ui/PresetEditor.cpp
+++ b/src/noggit/ui/tools/PresetEditor/Ui/PresetEditor.cpp
@@ -8,6 +8,7 @@
 #include <noggit/ui/FontNoggit.hpp>
 #include <noggit/ui/tools/PreviewRenderer/PreviewRenderer.hpp>
 #include <noggit/World.h>
+#include <noggit/database/ClientDatabase.h>

 #include <blizzard-database-library/include/BlizzardDatabase.h>

@@ -85,7 +86,8 @@ PresetEditorWidget::PresetEditorWidget(std::shared_ptr<Project::NoggitProject> p
  ui->worldSelector->setItemData(0, QVariant(-1));

  const auto& table = std::string("Map");
-  auto mapTable = _project->ClientDatabase->LoadTable(table, readFileAsIMemStream);
+
+  auto mapTable = ClientDatabase::getTable(table);

  int count = 1;
  auto iterator = mapTable.Records();
@@ -108,7 +110,6 @@ PresetEditorWidget::PresetEditorWidget(std::shared_ptr<Project::NoggitProject> p

      count++;
  }
-  _project->ClientDatabase->UnloadTable("Map");


  // Handle minimap widget
--- a/src/noggit/ui/windows/noggitWindow/NoggitWindow.cpp
+++ b/src/noggit/ui/windows/noggitWindow/NoggitWindow.cpp
@@ -21,6 +21,8 @@
 #include <noggit/ui/windows/settingsPanel/SettingsPanel.h>
 #include <noggit/uid_storage.hpp>
 #include <noggit/World.h>
+#include <noggit/database/SqlUIDStorage.h>
+#include <noggit/database/ClientDatabase.h>

 #include <string>
 #include <blizzard-archive-library/include/Exception.hpp>
@@ -43,16 +45,11 @@
 #include <QtWidgets/QTabWidget>
 #include <QtWidgets/QVBoxLayout>
 #include <QtWidgets/QWidget>
+#include <QtCore/QSettings>

 #include <chrono>
 #include <sstream>

-#ifdef USE_MYSQL_UID_STORAGE
-#include <mysql/mysql.h>
-
-#include <QtCore/QSettings>
-#endif
-
 #include "revision.h"

 #include "ui_TitleBar.h"
@@ -81,7 +78,28 @@ namespace Noggit::Ui::Windows
    }
    else
    {
-        LogError << "NoggitWindow() : Unsupported project version, skipping loading DBCs." << std::endl;
+      assert(false); // TODO
+      LogError << "NoggitWindow() : Unsupported project version, skipping loading DBCs." << std::endl;
+    }
+
+    _settings = new settings(this);
+
+    QSettings settings;
+    // connect to databases
+    bool use_mysql = settings.value("project/mysql/enabled", false).toBool();
+
+    ClientDatabase::setDatabaseMode(use_mysql ? DatabaseMode::Sql : DatabaseMode::ClientStorage);
+
+    if (use_mysql)
+    {
+      auto host = settings.value("project/mysql/server").toString();
+      auto port = settings.value("project/mysql/port", "3306").toInt();
+      auto db_name = settings.value("project/mysql/db").toString();  // schema name
+      auto user = settings.value("project/mysql/user").toString();
+      auto pass = settings.value("project/mysql/pwd").toString();
+
+      auto& db_manager = Sql::SqlDatabaseManager::instance();
+      db_manager.initializeDbConnection(Sql::SQLDbType::Noggit, host, port, db_name, user, pass);
    }

    setCentralWidget(_null_widget);
@@ -90,12 +108,9 @@ namespace Noggit::Ui::Windows
    setDockOptions(AnimatedDocks | AllowNestedDocks | AllowTabbedDocks | GroupedDragging);

    _about = new about(this);
-    _settings = new settings(this);

    _menuBar = menuBar();

-    QSettings settings;
-
    if (!settings.value("systemWindowFrame", true).toBool())
    {
      QWidget* widget = new QWidget(this);
@@ -159,16 +174,9 @@ namespace Noggit::Ui::Windows

    unsigned int world_map_id = getWorld()->getMapID();

-#ifdef USE_MYSQL_UID_STORAGE
    bool use_mysql = settings.value("project/mysql/enabled", false).toBool();

-    bool valid_conn = false;
-    if (use_mysql)
-    {
-        valid_conn = mysql::testConnection(true);
-    }
-
-    if ((valid_conn && mysql::hasMaxUIDStoredDB(world_map_id))
+    if ((Noggit::Sql::SqlUIDStorage::hasMaxUIDStoredDB(world_map_id))
      || uid_storage::hasMaxUIDStored(world_map_id)
       )
    {
@@ -176,7 +184,8 @@ namespace Noggit::Ui::Windows
      getWorld()->mapIndex.loadMaxUID();
      enterMapAt(pos, camera_pitch, camera_yaw, uid_fix_mode::none, from_bookmark);
    }
-#else
+    // old if no mysql block
+    /*
    if (uid_storage::hasMaxUIDStored(world_map_id))
    {
      if (settings.value("uid_startup_check", true).toBool())
@@ -187,8 +196,7 @@ namespace Noggit::Ui::Windows
        getWorld()->mapIndex.loadMaxUID();
        enterMapAt(pos, camera_pitch, camera_yaw, uid_fix_mode::none, from_bookmark);
      }
-    }
-#endif
+    }*/
    else
    {
      auto uid_fix_window(new UidFixWindow(pos, camera_pitch, camera_yaw));
--- a/src/noggit/ui/windows/noggitWindow/components/BuildMapListComponent.cpp
+++ b/src/noggit/ui/windows/noggitWindow/components/BuildMapListComponent.cpp
@@ -7,6 +7,7 @@
 #include <noggit/ui/windows/noggitWindow/NoggitWindow.hpp>
 #include <noggit/ui/windows/noggitWindow/widgets/MapListItem.hpp>
 #include <noggit/World.h>
+#include <noggit/database/ClientDatabase.h>

 #include <blizzard-database-library/include/BlizzardDatabase.h>

@@ -25,7 +26,7 @@ void BuildMapListComponent::buildMapList(Noggit::Ui::Windows::NoggitWindow* pare
  }

  const auto& table = std::string("Map");
-  auto map_table = parent->_project->ClientDatabase->LoadTable(table, readFileAsIMemStream);
+  auto map_table = ClientDatabase::getTable(table);

  auto iterator = map_table.Records();
  auto pinned_maps = std::vector<Widget::MapListData>();
@@ -126,6 +127,4 @@ void BuildMapListComponent::buildMapList(Noggit::Ui::Windows::NoggitWindow* pare
    item->setData(Qt::UserRole, QVariant(map.map_id));
    parent->_continents_table->setItemWidget(item, map_list_item);
  }
-
-  parent->_project->ClientDatabase->UnloadTable(table);
 }
--- a/src/noggit/ui/windows/settingsPanel/SettingsPanel.cpp
+++ b/src/noggit/ui/windows/settingsPanel/SettingsPanel.cpp
@@ -3,6 +3,7 @@
 #include <noggit/Log.h>
 #include <noggit/ui/FramelessWindow.hpp>
 #include <noggit/ui/windows/settingsPanel/SettingsPanel.h>
+#include <noggit/database/SqlDatabaseManager.h>

 #include <QDir>
 #include <QtCore/QSettings>
@@ -10,13 +11,10 @@
 #include <QtWidgets/QFileDialog>
 #include <QtWidgets/QPushButton>

-#ifdef USE_MYSQL_UID_STORAGE
-#include <mysql/mysql.h>
-#endif
-
 #include <ui_SettingsPanel.h>

 #include <sstream>
+#include <QMessageBox>


 namespace Noggit
@@ -79,11 +77,9 @@ namespace Noggit
      );


-#ifdef USE_MYSQL_UID_STORAGE
      ui->MySQL_box->setEnabled(true);
      ui->MySQL_box->setCheckable(true);
      ui->mysql_warning->setVisible(false);
-#endif

      ui->_theme->addItem("System");

@@ -170,10 +166,55 @@ namespace Noggit

      connect(ui->mysql_connect_test, &QPushButton::clicked, [this]
          {
-              save_changes();
-              #ifdef USE_MYSQL_UID_STORAGE
-              mysql::testConnection();
-              #endif
+              // test button for noggit db only !
+
+              // save_changes(); // old method saved and only used qsetting
+
+              auto& db_manager = Sql::SqlDatabaseManager::instance();
+
+              if (!db_manager.isDriverAvailable())
+              {
+                qDebug() << "Available drivers:" << QSqlDatabase::drivers();
+
+                // SPECIAL MISSING/currupted driver DLL POPUP
+                QMessageBox prompt;
+                prompt.setWindowFlag(Qt::WindowStaysOnTopHint);
+                prompt.setIcon(QMessageBox::Critical);
+                prompt.setText("mySQL driver could not be loaded.");
+                prompt.setWindowTitle("Noggit Database Error");
+
+                std::stringstream promptText;
+                promptText << "Make sure <Noggit/sqldrivers/qsqlmysql.dll>  exists.\n";
+                promptText << "If not, it can be downloaded at https://github.com/thecodemonkey86/qt_mysql_driver/releases \n";
+                promptText << "Noggit Qt version :" << QT_VERSION_STR;
+
+                prompt.setInformativeText("Make sure Noggit/sqldrivers/qsqlmysql.dll exists."
+                "If not, it can be downloaded at ");
+                prompt.exec();
+                return;
+              }
+
+              QString host = ui->_mysql_server_field->text();
+              QString user = ui->_mysql_user_field->text();
+              QString pass = ui->_mysql_pwd_field->text();
+              QString db_name = ui->_mysql_db_field->text();
+              int port = ui->_mysql_port_field->value();
+
+              // test temporary connection from text fields
+              bool test1 = db_manager.initializeDbConnection(Sql::SQLDbType::Noggit, host, port, db_name, user, pass);
+              bool test2 = db_manager.testConnectionWithPopup(Sql::SQLDbType::Noggit, false);
+
+              // reconnect to saved db from settings
+              QSettings settings;
+              host = settings.value("project/mysql/server").toString();
+              port = settings.value("project/mysql/port", "3306").toInt();
+              db_name = settings.value("project/mysql/db").toString();  // schema name
+              user = settings.value("project/mysql/user").toString();
+              pass = settings.value("project/mysql/pwd").toString();
+
+              bool test3 = db_manager.initializeDbConnection(Sql::SQLDbType::Noggit, host, port, db_name, user, pass);
+              assert(db_manager.testConnection(Sql::SQLDbType::Noggit));
+
          }
      );

@@ -234,15 +275,13 @@ namespace Noggit
      ui->_wmo_aabb->setChecked(_settings->value("render/wmo_aabb", false).toBool());
      ui->_wmo_group_bounds->setChecked(_settings->value("render/wmo_groups_bounds", false).toBool());

-
-#ifdef USE_MYSQL_UID_STORAGE
      ui->MySQL_box->setChecked (_settings->value ("project/mysql/enabled").toBool());

      auto server_str = _settings->value("project/mysql/server", "127.0.0.1").toString();
-      auto user_str = _settings->value("project/mysql/user", "127.0.0.1").toString();
-      auto pwd_str = _settings->value("project/mysql/pwd", "127.0.0.1").toString();
-      auto db_str = _settings->value("project/mysql/db", "127.0.0.1").toString();
-      auto port_int = _settings->value("project/mysql/port", "127.0.0.1").toInt();
+      auto user_str = _settings->value("project/mysql/user", "root").toString();
+      auto pwd_str = _settings->value("project/mysql/pwd", "root").toString();
+      auto db_str = _settings->value("project/mysql/db", "noggit").toString();
+      auto port_int = _settings->value("project/mysql/port", "3306").toInt();

      // set some default
      if (server_str.isEmpty())
@@ -261,7 +300,6 @@ namespace Noggit
      ui->_mysql_pwd_field->setText (pwd_str);
      ui->_mysql_db_field->setText (db_str);
      ui->_mysql_port_field->setValue (port_int);
-#endif

      bool wireframe_type = _settings->value("wireframe/type", false).toBool();

@@ -308,14 +346,12 @@ namespace Noggit
      _settings->setValue("modern_features", ui->_modern_features->isChecked());
      _settings->setValue("use_mclq_liquids_export", ui->_use_mclq_liquids_export->isChecked());

-#ifdef USE_MYSQL_UID_STORAGE
      _settings->setValue ("project/mysql/enabled", ui->MySQL_box->isChecked());
      _settings->setValue ("project/mysql/server", ui->_mysql_server_field->text());
      _settings->setValue ("project/mysql/user", ui->_mysql_user_field->text());
      _settings->setValue ("project/mysql/pwd", ui->_mysql_pwd_field->text());
      _settings->setValue ("project/mysql/db", ui->_mysql_db_field->text());
      _settings->setValue ("project/mysql/port", ui->_mysql_port_field->text());
-#endif

      _settings->setValue("wireframe/type", ui->radio_wire_cursor->isChecked());
      _settings->setValue("wireframe/radius", ui->_wireframe_radius->value());
@@ -338,6 +374,18 @@ namespace Noggit

      _settings->sync();

+      // reinitialize db on save
+      auto& db_manager = Sql::SqlDatabaseManager::instance();
+      QString host = _settings->value("project/mysql/server").toString();
+      int port = _settings->value("project/mysql/port", "3306").toInt();
+      QString db_name = _settings->value("project/mysql/db").toString();
+      QString user = _settings->value("project/mysql/user").toString();
+      QString pass = _settings->value("project/mysql/pwd").toString();
+
+      bool test1 = db_manager.initializeDbConnection(Sql::SQLDbType::Noggit, host, port, db_name, user, pass);
+      bool test2 = db_manager.testConnectionWithPopup(Sql::SQLDbType::Noggit, true);
+
+
      // calls MapView::onSettingsSave()
      emit saved();
    }
Author	SHA1	Message	Date
T1ti	2fe2a2644d	Fix color calculation for variable 'b' in WMO.cpp cherry pick from `0347333f95`	2025-10-23 23:34:54 +02:00
T1ti	061c18e069	optimize alphamap updates	2025-10-16 22:50:07 +02:00
T1ti	0d600577da	optimize dbc reader	2025-09-24 05:04:11 +02:00
T1ti	6a4e06a76d	massively speed up SQL	2025-09-23 05:48:37 +02:00
T1ti	578c83a72c	rewrite ClientDatabase and move to new sql api	2025-09-23 01:31:41 +02:00
T1ti	81af05eaaa	rename database classes	2025-09-19 21:37:39 +02:00
T1ti	8f0865cc83	bulk sql insert update	2025-09-19 19:40:15 +02:00
T1ti	c5b1ca786c	fix dbc shared reader bug when not unloading, and optimize bulk query	2025-09-19 17:35:43 +02:00
T1ti	77bc08b70b	Merge branch 'ground_effects_editor' of https://gitlab.com/T1ti/noggit-red into ground_effects_editor # Conflicts: # .gitignore # CMakeLists.txt	2025-09-19 03:50:47 +02:00
T1ti	f66c525f90	port from mysql conenctor to QTSQL module add new classes to manage connections begin work on generic database for client and db	2025-09-19 03:44:34 +02:00
T1ti	8c6b89508b	Merge branch 'server' into 'ground_effects_editor' Streamline build + gitignore fix See merge request T1ti/noggit-red!15	2025-09-16 18:17:24 +00:00
dwg	f8b98f3b6c	Make configuring the build more straight-forward by allowing the user to just specify the base path to the mysql libraries. These two new cmake variables where added: MYSQL_ROOT - Points to something like "C:\Program Files\MySQL\MySQL Server 8.0" MYSQLCPPCONN_ROOT - Points to something like "C:\Program Files\MySQL\mysql-connector-c++-8.1.0-winx64" This will also make copying the required dlls easier, but that's outside the scope of this commit	2025-09-16 20:03:36 +02:00
dwg	af326d1bc2	Fix gitignore	2025-09-16 19:30:11 +02:00