diff options
| -rw-r--r-- | Tools/MCADefrag/.gitignore | 1 | ||||
| -rw-r--r-- | Tools/MCADefrag/CMakeLists.txt | 144 | ||||
| -rw-r--r-- | Tools/MCADefrag/Globals.cpp | 10 | ||||
| -rw-r--r-- | Tools/MCADefrag/Globals.h | 229 | ||||
| -rw-r--r-- | Tools/MCADefrag/MCADefrag.cpp | 303 | ||||
| -rw-r--r-- | Tools/MCADefrag/MCADefrag.h | 112 |
6 files changed, 799 insertions, 0 deletions
diff --git a/Tools/MCADefrag/.gitignore b/Tools/MCADefrag/.gitignore new file mode 100644 index 000000000..44a3e1f48 --- /dev/null +++ b/Tools/MCADefrag/.gitignore @@ -0,0 +1 @@ +*.mca diff --git a/Tools/MCADefrag/CMakeLists.txt b/Tools/MCADefrag/CMakeLists.txt new file mode 100644 index 000000000..7296b8ddc --- /dev/null +++ b/Tools/MCADefrag/CMakeLists.txt @@ -0,0 +1,144 @@ + +cmake_minimum_required (VERSION 2.6) + +project (MCADefrag) + + + +macro(add_flags_cxx FLAGS) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${FLAGS}") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${FLAGS}") + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${FLAGS}") + set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${FLAGS}") + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${FLAGS}") + set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${FLAGS}") +endmacro() + + + + +# Add the preprocessor macros used for distinguishing between debug and release builds (CMake does this automatically for MSVC): +if (NOT MSVC) + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_DEBUG") + set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -D_DEBUG") + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -DNDEBUG") + set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -DNDEBUG") +endif() + + + +if(MSVC) + # Make build use multiple threads under MSVC: + add_flags_cxx("/MP") + + # Make release builds use link-time code generation: + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /GL") + set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} /GL") + set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} /LTCG") + set(CMAKE_SHARED_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /LTCG") + set(CMAKE_MODULE_LINKER_FLAGS_RELEASE "${CMAKE_MODULE_LINKER_FLAGS_RELEASE} /LTCG") +elseif(APPLE) + #on os x clang adds pthread for us but we need to add it for gcc + if (NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + add_flags_cxx("-pthread") + endif() +else() + # Let gcc / clang know that we're compiling a multi-threaded app: + add_flags_cxx("-pthread") +endif() + + + + +# Use static CRT in MSVC builds: +if (MSVC) + string(REPLACE "/MD" "/MT" CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}") + string(REPLACE "/MD" "/MT" CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}") + string(REPLACE "/MDd" "/MTd" CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG}") + string(REPLACE "/MDd" "/MTd" CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG}") +endif() + + + + +# Set include paths to the used libraries: +include_directories("../../lib") +include_directories("../../src") + + + +function(flatten_files arg1) + set(res "") + foreach(f ${${arg1}}) + get_filename_component(f ${f} ABSOLUTE) + list(APPEND res ${f}) + endforeach() + set(${arg1} "${res}" PARENT_SCOPE) +endfunction() + + +# Include the libraries: +file(GLOB ZLIB_SRC "../../lib/zlib/*.c") +file(GLOB ZLIB_HDR "../../lib/zlib/*.h") +flatten_files(ZLIB_SRC) +flatten_files(ZLIB_HDR) +source_group("ZLib" FILES ${ZLIB_SRC} ${ZLIB_HDR}) + + +# Include the shared files: +set(SHARED_SRC + ../../src/StringCompression.cpp + ../../src/StringUtils.cpp + ../../src/Log.cpp + ../../src/MCLogger.cpp +) +set(SHARED_HDR + ../../src/ByteBuffer.h + ../../src/StringUtils.h + ../../src/Log.h + ../../src/MCLogger.h +) +set(SHARED_OSS_SRC + ../../src/OSSupport/CriticalSection.cpp + ../../src/OSSupport/File.cpp + ../../src/OSSupport/IsThread.cpp + ../../src/OSSupport/Timer.cpp +) +set(SHARED_OSS_HDR + ../../src/OSSupport/CriticalSection.h + ../../src/OSSupport/File.h + ../../src/OSSupport/IsThread.h + ../../src/OSSupport/Timer.h +) +flatten_files(SHARED_SRC) +flatten_files(SHARED_HDR) +flatten_files(SHARED_OSS_SRC) +flatten_files(SHARED_OSS_HDR) +source_group("Shared" FILES ${SHARED_SRC} ${SHARED_HDR}) +source_group("Shared\\OSSupport" FILES ${SHARED_OSS_SRC} ${SHARED_OSS_HDR}) + + + +# Include the main source files: +set(SOURCES + MCADefrag.cpp + Globals.cpp +) +set(HEADERS + MCADefrag.h + Globals.h +) + +source_group("" FILES ${SOURCES} ${HEADERS}) + +add_executable(MCADefrag + ${SOURCES} + ${HEADERS} + ${SHARED_SRC} + ${SHARED_HDR} + ${SHARED_OSS_SRC} + ${SHARED_OSS_HDR} + ${ZLIB_SRC} + ${ZLIB_HDR} +) + diff --git a/Tools/MCADefrag/Globals.cpp b/Tools/MCADefrag/Globals.cpp new file mode 100644 index 000000000..13c6ae709 --- /dev/null +++ b/Tools/MCADefrag/Globals.cpp @@ -0,0 +1,10 @@ + +// Globals.cpp + +// This file is used for precompiled header generation in MSVC environments + +#include "Globals.h" + + + + diff --git a/Tools/MCADefrag/Globals.h b/Tools/MCADefrag/Globals.h new file mode 100644 index 000000000..6f4bbdc76 --- /dev/null +++ b/Tools/MCADefrag/Globals.h @@ -0,0 +1,229 @@ + +// Globals.h + +// This file gets included from every module in the project, so that global symbols may be introduced easily +// Also used for precompiled header generation in MSVC environments + + + + + +// Compiler-dependent stuff: +#if defined(_MSC_VER) + // MSVC produces warning C4481 on the override keyword usage, so disable the warning altogether + #pragma warning(disable:4481) + + // Disable some warnings that we don't care about: + #pragma warning(disable:4100) + + #define OBSOLETE __declspec(deprecated) + + // No alignment needed in MSVC + #define ALIGN_8 + #define ALIGN_16 + +#elif defined(__GNUC__) + + // TODO: Can GCC explicitly mark classes as abstract (no instances can be created)? + #define abstract + + // TODO: Can GCC mark virtual methods as overriding (forcing them to have a virtual function of the same signature in the base class) + #define override + + #define OBSOLETE __attribute__((deprecated)) + + #define ALIGN_8 __attribute__((aligned(8))) + #define ALIGN_16 __attribute__((aligned(16))) + + // Some portability macros :) + #define stricmp strcasecmp + +#else + + #error "You are using an unsupported compiler, you might need to #define some stuff here for your compiler" + + /* + // Copy and uncomment this into another #elif section based on your compiler identification + + // Explicitly mark classes as abstract (no instances can be created) + #define abstract + + // Mark virtual methods as overriding (forcing them to have a virtual function of the same signature in the base class) + #define override + + // Mark functions as obsolete, so that their usage results in a compile-time warning + #define OBSOLETE + + // Mark types / variables for alignment. Do the platforms need it? + #define ALIGN_8 + #define ALIGN_16 + */ + +#endif + + + + + +// Integral types with predefined sizes: +typedef long long Int64; +typedef int Int32; +typedef short Int16; + +typedef unsigned long long UInt64; +typedef unsigned int UInt32; +typedef unsigned short UInt16; + +typedef unsigned char Byte; + + + + + +// A macro to disallow the copy constructor and operator= functions +// This should be used in the private: declarations for any class that shouldn't allow copying itself +#define DISALLOW_COPY_AND_ASSIGN(TypeName) \ + TypeName(const TypeName &); \ + void operator=(const TypeName &) + +// A macro that is used to mark unused function parameters, to avoid pedantic warnings in gcc +#define UNUSED(X) (void)(X) + + + + +// OS-dependent stuff: +#ifdef _WIN32 + #define WIN32_LEAN_AND_MEAN + #include <Windows.h> + #include <winsock2.h> + #include <ws2tcpip.h> + + // Windows SDK defines min and max macros, messing up with our std::min and std::max usage + #undef min + #undef max + + // Windows SDK defines GetFreeSpace as a constant, probably a Win16 API remnant + #ifdef GetFreeSpace + #undef GetFreeSpace + #endif // GetFreeSpace + + #define SocketError WSAGetLastError() +#else + #include <sys/types.h> + #include <sys/stat.h> // for mkdir + #include <sys/time.h> + #include <sys/socket.h> + #include <netinet/in.h> + #include <arpa/inet.h> + #include <netdb.h> + #include <time.h> + #include <dirent.h> + #include <errno.h> + #include <iostream> + #include <unistd.h> + + #include <cstdio> + #include <cstring> + #include <pthread.h> + #include <semaphore.h> + #include <errno.h> + #include <fcntl.h> + + typedef int SOCKET; + enum + { + INVALID_SOCKET = -1, + }; + #define closesocket close + #define SocketError errno +#if !defined(ANDROID_NDK) + #include <tr1/memory> +#endif +#endif + +#if !defined(ANDROID_NDK) + #define USE_SQUIRREL +#endif + +#if defined(ANDROID_NDK) + #define FILE_IO_PREFIX "/sdcard/mcserver/" +#else + #define FILE_IO_PREFIX "" +#endif + + + + + +// CRT stuff: +#include <assert.h> +#include <stdio.h> +#include <math.h> +#include <stdarg.h> +#include <time.h> + + + + + +// STL stuff: +#include <vector> +#include <list> +#include <deque> +#include <string> +#include <map> +#include <algorithm> +#include <memory> + + + + + +// Common headers (without macros): +#include "StringUtils.h" +#include "OSSupport/CriticalSection.h" +#include "OSSupport/IsThread.h" +#include "OSSupport/File.h" + + + + + +// Common definitions: + +/// Evaluates to the number of elements in an array (compile-time!) +#define ARRAYCOUNT(X) (sizeof(X) / sizeof(*(X))) + +/// Allows arithmetic expressions like "32 KiB" (but consider using parenthesis around it, "(32 KiB)" ) +#define KiB * 1024 +#define MiB * 1024 * 1024 + +/// Faster than (int)floorf((float)x / (float)div) +#define FAST_FLOOR_DIV( x, div ) ( (x) < 0 ? (((int)x / div) - 1) : ((int)x / div) ) + +// Own version of assert() that writes failed assertions to the log for review +#ifdef NDEBUG + #define ASSERT(x) ((void)0) +#else + #define ASSERT assert +#endif + +// Pretty much the same as ASSERT() but stays in Release builds +#define VERIFY( x ) ( !!(x) || ( LOGERROR("Verification failed: %s, file %s, line %i", #x, __FILE__, __LINE__ ), exit(1), 0 ) ) + + + + + +/// A generic interface used mainly in ForEach() functions +template <typename Type> class cItemCallback +{ +public: + /// Called for each item in the internal list; return true to stop the loop, or false to continue enumerating + virtual bool Item(Type * a_Type) = 0; +} ; + + + + diff --git a/Tools/MCADefrag/MCADefrag.cpp b/Tools/MCADefrag/MCADefrag.cpp new file mode 100644 index 000000000..2f24469c9 --- /dev/null +++ b/Tools/MCADefrag/MCADefrag.cpp @@ -0,0 +1,303 @@ + +// MCADefrag.cpp + +// Implements the main app entrypoint and the cMCADefrag class representing the entire app + +#include "Globals.h" +#include "MCADefrag.h" +#include "MCLogger.h" + + + + + +int main(int argc, char ** argv) +{ + new cMCLogger(Printf("Defrag_%08x.log", time(NULL))); + cMCADefrag Defrag; + if (!Defrag.Init(argc, argv)) + { + return 1; + } + + Defrag.Run(); + + return 0; +} + + + + + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// cMCADefrag: + +cMCADefrag::cMCADefrag(void) : + m_NumThreads(1) +{ +} + + + + + +bool cMCADefrag::Init(int argc, char ** argv) +{ + // Nothing needed yet + return true; +} + + + + + +void cMCADefrag::Run(void) +{ + // Fill the queue with MCA files + m_Queue = cFile::GetFolderContents("."); + + // Start the processing threads: + for (int i = 0; i < m_NumThreads; i++) + { + StartThread(); + } + + // Wait for all the threads to finish: + while (!m_Threads.empty()) + { + m_Threads.front()->Wait(); + delete m_Threads.front(); + m_Threads.pop_front(); + } +} + + + + +void cMCADefrag::StartThread(void) +{ + cThread * Thread = new cThread(*this); + m_Threads.push_back(Thread); + Thread->Start(); +} + + + + + +AString cMCADefrag::GetNextFileName(void) +{ + cCSLock Lock(m_CS); + if (m_Queue.empty()) + { + return AString(); + } + AString res = m_Queue.back(); + m_Queue.pop_back(); + return res; +} + + + + + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// cMCADefrag::cThread: + +cMCADefrag::cThread::cThread(cMCADefrag & a_Parent) : + super("MCADefrag thread"), + m_Parent(a_Parent) +{ +} + + + + + +void cMCADefrag::cThread::Execute(void) +{ + for (;;) + { + AString FileName = m_Parent.GetNextFileName(); + if (FileName.empty()) + { + return; + } + ProcessFile(FileName); + } +} + + + + + +void cMCADefrag::cThread::ProcessFile(const AString & a_FileName) +{ + // Filter out non-MCA files: + if ((a_FileName.length() < 4) || (a_FileName.substr(a_FileName.length() - 4, 4) != ".mca")) + { + return; + } + LOGINFO("%s", a_FileName.c_str()); + + // Open input and output files: + AString OutFileName = a_FileName + ".new"; + cFile In, Out; + if (!In.Open(a_FileName, cFile::fmRead)) + { + LOGWARNING("Cannot open file %s for reading, skipping file.", a_FileName.c_str()); + return; + } + if (!Out.Open(OutFileName.c_str(), cFile::fmWrite)) + { + LOGWARNING("Cannot open file %s for writing, skipping file.", OutFileName.c_str()); + return; + } + + // Read the Locations and Timestamps from the input file: + Byte Locations[4096]; + UInt32 Timestamps[1024]; + if (In.Read(Locations, sizeof(Locations)) != sizeof(Locations)) + { + LOGWARNING("Cannot read Locations in file %s, skipping file.", a_FileName.c_str()); + return; + } + if (In.Read(Timestamps, sizeof(Timestamps)) != sizeof(Timestamps)) + { + LOGWARNING("Cannot read Timestamps in file %s, skipping file.", a_FileName.c_str()); + return; + } + + // Write dummy Locations to the Out file (will be overwritten once the correct ones are known) + if (Out.Write(Locations, sizeof(Locations)) != sizeof(Locations)) + { + LOGWARNING("Cannot write Locations to file %s, skipping file.", OutFileName.c_str()); + return; + } + m_CurrentSectorOut = 2; + + // Write a copy of the Timestamps into the Out file: + if (Out.Write(Timestamps, sizeof(Timestamps)) != sizeof(Timestamps)) + { + LOGWARNING("Cannot write Timestamps to file %s, skipping file.", OutFileName.c_str()); + return; + } + + // Process each chunk: + for (size_t i = 0; i < 1024; i++) + { + size_t idx = i * 4; + if ( + (Locations[idx] == 0) && + (Locations[idx + 1] == 0) && + (Locations[idx + 2] == 0) && + (Locations[idx + 3] == 0) + ) + { + // Chunk not present + continue; + } + if (!ReadChunk(In, Locations + idx)) + { + LOGWARNING("Cannot read chunk #%d from file %s. Skipping file.", i, a_FileName.c_str()); + return; + } + if (!WriteChunk(Out, Locations + idx)) + { + LOGWARNING("Cannot write chunk #%d to file %s. Skipping file.", i, OutFileName.c_str()); + return; + } + } + + // Write the new Locations into the MCA header: + Out.Seek(0); + if (Out.Write(Locations, sizeof(Locations)) != sizeof(Locations)) + { + LOGWARNING("Cannot write updated Locations to file %s, skipping file.", OutFileName.c_str()); + return; + } + + // Close the files, delete orig, rename new: + In.Close(); + Out.Close(); + cFile::Delete(a_FileName); + cFile::Rename(OutFileName, a_FileName); +} + + + + + +bool cMCADefrag::cThread::ReadChunk(cFile & a_File, const Byte * a_LocationRaw) +{ + int SectorNum = (a_LocationRaw[0] << 16) | (a_LocationRaw[1] << 8) | a_LocationRaw[2]; + int SizeInSectors = a_LocationRaw[3] * (4 KiB); + if (a_File.Seek(SectorNum * (4 KiB)) < 0) + { + LOGWARNING("Failed to seek to chunk data - file pos %llu (%d KiB, %.02f MiB)!", (Int64)SectorNum * (4 KiB), SectorNum * 4, ((double)SectorNum) / 256); + return false; + } + + // Read the exact size: + Byte Buf[4]; + if (a_File.Read(Buf, 4) != 4) + { + LOGWARNING("Failed to read chunk data length"); + return false; + } + m_CompressedChunkDataSize = (Buf[0] << 24) | (Buf[1] << 16) | (Buf[2] << 8) | Buf[3]; + if (m_CompressedChunkDataSize > SizeInSectors) + { + LOGWARNING("Invalid chunk data - SizeInSectors (%d) smaller that RealSize (%d)", SizeInSectors, m_CompressedChunkDataSize); + return false; + } + + // Read the data: + if (a_File.Read(m_CompressedChunkData, m_CompressedChunkDataSize) != m_CompressedChunkDataSize) + { + LOGWARNING("Failed to read chunk data!"); + return false; + } + + // TODO: Uncompress the data if recompression is active + + return true; +} + + + + + +bool cMCADefrag::cThread::WriteChunk(cFile & a_File, Byte * a_LocationRaw) +{ + // TODO: Recompress the data if recompression is active + + a_LocationRaw[0] = m_CurrentSectorOut >> 16; + a_LocationRaw[1] = (m_CurrentSectorOut >> 8) & 0xff; + a_LocationRaw[2] = m_CurrentSectorOut & 0xff; + a_LocationRaw[3] = (m_CompressedChunkDataSize + (4 KiB) - 1) / (4 KiB); + + // Write the data length: + Byte Buf[4]; + Buf[0] = m_CompressedChunkDataSize >> 24; + Buf[1] = (m_CompressedChunkDataSize >> 16) & 0xff; + Buf[2] = (m_CompressedChunkDataSize >> 8) & 0xff; + Buf[3] = m_CompressedChunkDataSize & 0xff; + if (a_File.Write(Buf, 4) != 4) + { + LOGWARNING("Failed to write chunk length!"); + return false; + } + + // Write the data: + if (a_File.Write(m_CompressedChunkData, m_CompressedChunkDataSize) != m_CompressedChunkDataSize) + { + LOGWARNING("Failed to write chunk data!"); + return false; + } + return true; +} + + + + diff --git a/Tools/MCADefrag/MCADefrag.h b/Tools/MCADefrag/MCADefrag.h new file mode 100644 index 000000000..0801d1b5d --- /dev/null +++ b/Tools/MCADefrag/MCADefrag.h @@ -0,0 +1,112 @@ + +// MCADefrag.h + +// Interfaces to the cMCADefrag class encapsulating the entire app + + + + + +#pragma once + + + + + + +class cMCADefrag +{ +public: + enum + { + MAX_COMPRESSED_CHUNK_DATA_SIZE = (1 MiB), + MAX_RAW_CHUNK_DATA_SIZE = (100 MiB), + } ; + + cMCADefrag(void); + + /** Reads the cmdline params and initializes the app. + Returns true if the app should continue, false if not. */ + bool Init(int argc, char ** argv); + + /** Runs the entire app. */ + void Run(void); + +protected: + /** A single thread processing MCA files from the queue */ + class cThread : + public cIsThread + { + typedef cIsThread super; + + public: + cThread(cMCADefrag & a_Parent); + + protected: + cMCADefrag & m_Parent; + + /** The current compressed chunk data. Valid after a successful ReadChunk(). + This contains only the compression method byte and the compressed data, + but not the length preceding the data in the MCA file. */ + unsigned char m_CompressedChunkData[MAX_COMPRESSED_CHUNK_DATA_SIZE]; + + /** Size of the actual current compressed chunk data. */ + int m_CompressedChunkDataSize; + + /** The current raw chunk data. Valid after a successful ReadChunk(), if recompression is active. */ + unsigned char m_RawChunkData[MAX_RAW_CHUNK_DATA_SIZE]; + + /** Size of the actual current raw chunk data. */ + int m_RawChunkDataSize; + + /** Number of the sector where the next chunk will be written by WriteChunk(). */ + int m_CurrentSectorOut; + + + /** Processes the specified file. */ + void ProcessFile(const AString & a_FileName); + + /** Reads the chunk data into m_CompressedChunkData. + Calls DecompressChunkData() if recompression is active. + a_LocationRaw is the pointer to the first byte of the Location data in the MCA header. + Returns true if successful. */ + bool ReadChunk(cFile & a_File, const Byte * a_LocationRaw); + + /** Writes the chunk data from m_CompressedData or m_RawChunkData (performing recompression) into file. + Calls CompressChunkData() for the actual compression, if recompression is active. + a_LocationRaw is the pointer to the first byte of the Location data to be put into the MCA header, + the chunk's location is stored in that memory area. Updates m_CurrentSectorOut. + Returns true if successful. */ + bool WriteChunk(cFile & a_File, Byte * a_LocationRaw); + + // cIsThread overrides: + virtual void Execute(void) override; + } ; + + typedef std::list<cThread *> cThreads; + + + /** The mutex protecting m_Files agains multithreaded access. */ + cCriticalSection m_CS; + + /** The queue of MCA files to be processed by the threads. Protected by m_CS. */ + AStringVector m_Queue; + + /** List of threads that the server has running. */ + cThreads m_Threads; + + /** The number of threads that should be started. Configurable on the command line. */ + int m_NumThreads; + + + /** Starts a new processing thread and adds it to cThreads. */ + void StartThread(void); + + /** Retrieves one file from the queue (and removes it from the queue). + Returns an empty string when queue empty. */ + AString GetNextFileName(void); +} ; + + + + |