From 05590fb91d7f4bd2178847cc56ae31699f09ddd2 Mon Sep 17 00:00:00 2001 From: madmaxoft Date: Thu, 13 Feb 2014 11:47:20 +0100 Subject: MCADefrag: Initial implementation. Partially implements #639. This only defragments the chunks, without recompressing them. --- Tools/MCADefrag/MCADefrag.cpp | 303 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 303 insertions(+) create mode 100644 Tools/MCADefrag/MCADefrag.cpp (limited to 'Tools/MCADefrag/MCADefrag.cpp') diff --git a/Tools/MCADefrag/MCADefrag.cpp b/Tools/MCADefrag/MCADefrag.cpp new file mode 100644 index 000000000..2f24469c9 --- /dev/null +++ b/Tools/MCADefrag/MCADefrag.cpp @@ -0,0 +1,303 @@ + +// MCADefrag.cpp + +// Implements the main app entrypoint and the cMCADefrag class representing the entire app + +#include "Globals.h" +#include "MCADefrag.h" +#include "MCLogger.h" + + + + + +int main(int argc, char ** argv) +{ + new cMCLogger(Printf("Defrag_%08x.log", time(NULL))); + cMCADefrag Defrag; + if (!Defrag.Init(argc, argv)) + { + return 1; + } + + Defrag.Run(); + + return 0; +} + + + + + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// cMCADefrag: + +cMCADefrag::cMCADefrag(void) : + m_NumThreads(1) +{ +} + + + + + +bool cMCADefrag::Init(int argc, char ** argv) +{ + // Nothing needed yet + return true; +} + + + + + +void cMCADefrag::Run(void) +{ + // Fill the queue with MCA files + m_Queue = cFile::GetFolderContents("."); + + // Start the processing threads: + for (int i = 0; i < m_NumThreads; i++) + { + StartThread(); + } + + // Wait for all the threads to finish: + while (!m_Threads.empty()) + { + m_Threads.front()->Wait(); + delete m_Threads.front(); + m_Threads.pop_front(); + } +} + + + + +void cMCADefrag::StartThread(void) +{ + cThread * Thread = new cThread(*this); + m_Threads.push_back(Thread); + Thread->Start(); +} + + + + + +AString cMCADefrag::GetNextFileName(void) +{ + cCSLock Lock(m_CS); + if (m_Queue.empty()) + { + return AString(); + } + AString res = m_Queue.back(); + m_Queue.pop_back(); + return res; +} + + + + + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// cMCADefrag::cThread: + +cMCADefrag::cThread::cThread(cMCADefrag & a_Parent) : + super("MCADefrag thread"), + m_Parent(a_Parent) +{ +} + + + + + +void cMCADefrag::cThread::Execute(void) +{ + for (;;) + { + AString FileName = m_Parent.GetNextFileName(); + if (FileName.empty()) + { + return; + } + ProcessFile(FileName); + } +} + + + + + +void cMCADefrag::cThread::ProcessFile(const AString & a_FileName) +{ + // Filter out non-MCA files: + if ((a_FileName.length() < 4) || (a_FileName.substr(a_FileName.length() - 4, 4) != ".mca")) + { + return; + } + LOGINFO("%s", a_FileName.c_str()); + + // Open input and output files: + AString OutFileName = a_FileName + ".new"; + cFile In, Out; + if (!In.Open(a_FileName, cFile::fmRead)) + { + LOGWARNING("Cannot open file %s for reading, skipping file.", a_FileName.c_str()); + return; + } + if (!Out.Open(OutFileName.c_str(), cFile::fmWrite)) + { + LOGWARNING("Cannot open file %s for writing, skipping file.", OutFileName.c_str()); + return; + } + + // Read the Locations and Timestamps from the input file: + Byte Locations[4096]; + UInt32 Timestamps[1024]; + if (In.Read(Locations, sizeof(Locations)) != sizeof(Locations)) + { + LOGWARNING("Cannot read Locations in file %s, skipping file.", a_FileName.c_str()); + return; + } + if (In.Read(Timestamps, sizeof(Timestamps)) != sizeof(Timestamps)) + { + LOGWARNING("Cannot read Timestamps in file %s, skipping file.", a_FileName.c_str()); + return; + } + + // Write dummy Locations to the Out file (will be overwritten once the correct ones are known) + if (Out.Write(Locations, sizeof(Locations)) != sizeof(Locations)) + { + LOGWARNING("Cannot write Locations to file %s, skipping file.", OutFileName.c_str()); + return; + } + m_CurrentSectorOut = 2; + + // Write a copy of the Timestamps into the Out file: + if (Out.Write(Timestamps, sizeof(Timestamps)) != sizeof(Timestamps)) + { + LOGWARNING("Cannot write Timestamps to file %s, skipping file.", OutFileName.c_str()); + return; + } + + // Process each chunk: + for (size_t i = 0; i < 1024; i++) + { + size_t idx = i * 4; + if ( + (Locations[idx] == 0) && + (Locations[idx + 1] == 0) && + (Locations[idx + 2] == 0) && + (Locations[idx + 3] == 0) + ) + { + // Chunk not present + continue; + } + if (!ReadChunk(In, Locations + idx)) + { + LOGWARNING("Cannot read chunk #%d from file %s. Skipping file.", i, a_FileName.c_str()); + return; + } + if (!WriteChunk(Out, Locations + idx)) + { + LOGWARNING("Cannot write chunk #%d to file %s. Skipping file.", i, OutFileName.c_str()); + return; + } + } + + // Write the new Locations into the MCA header: + Out.Seek(0); + if (Out.Write(Locations, sizeof(Locations)) != sizeof(Locations)) + { + LOGWARNING("Cannot write updated Locations to file %s, skipping file.", OutFileName.c_str()); + return; + } + + // Close the files, delete orig, rename new: + In.Close(); + Out.Close(); + cFile::Delete(a_FileName); + cFile::Rename(OutFileName, a_FileName); +} + + + + + +bool cMCADefrag::cThread::ReadChunk(cFile & a_File, const Byte * a_LocationRaw) +{ + int SectorNum = (a_LocationRaw[0] << 16) | (a_LocationRaw[1] << 8) | a_LocationRaw[2]; + int SizeInSectors = a_LocationRaw[3] * (4 KiB); + if (a_File.Seek(SectorNum * (4 KiB)) < 0) + { + LOGWARNING("Failed to seek to chunk data - file pos %llu (%d KiB, %.02f MiB)!", (Int64)SectorNum * (4 KiB), SectorNum * 4, ((double)SectorNum) / 256); + return false; + } + + // Read the exact size: + Byte Buf[4]; + if (a_File.Read(Buf, 4) != 4) + { + LOGWARNING("Failed to read chunk data length"); + return false; + } + m_CompressedChunkDataSize = (Buf[0] << 24) | (Buf[1] << 16) | (Buf[2] << 8) | Buf[3]; + if (m_CompressedChunkDataSize > SizeInSectors) + { + LOGWARNING("Invalid chunk data - SizeInSectors (%d) smaller that RealSize (%d)", SizeInSectors, m_CompressedChunkDataSize); + return false; + } + + // Read the data: + if (a_File.Read(m_CompressedChunkData, m_CompressedChunkDataSize) != m_CompressedChunkDataSize) + { + LOGWARNING("Failed to read chunk data!"); + return false; + } + + // TODO: Uncompress the data if recompression is active + + return true; +} + + + + + +bool cMCADefrag::cThread::WriteChunk(cFile & a_File, Byte * a_LocationRaw) +{ + // TODO: Recompress the data if recompression is active + + a_LocationRaw[0] = m_CurrentSectorOut >> 16; + a_LocationRaw[1] = (m_CurrentSectorOut >> 8) & 0xff; + a_LocationRaw[2] = m_CurrentSectorOut & 0xff; + a_LocationRaw[3] = (m_CompressedChunkDataSize + (4 KiB) - 1) / (4 KiB); + + // Write the data length: + Byte Buf[4]; + Buf[0] = m_CompressedChunkDataSize >> 24; + Buf[1] = (m_CompressedChunkDataSize >> 16) & 0xff; + Buf[2] = (m_CompressedChunkDataSize >> 8) & 0xff; + Buf[3] = m_CompressedChunkDataSize & 0xff; + if (a_File.Write(Buf, 4) != 4) + { + LOGWARNING("Failed to write chunk length!"); + return false; + } + + // Write the data: + if (a_File.Write(m_CompressedChunkData, m_CompressedChunkDataSize) != m_CompressedChunkDataSize) + { + LOGWARNING("Failed to write chunk data!"); + return false; + } + return true; +} + + + + -- cgit v1.2.3 From cd658e02e8dcb7503d8aa5d75c314dcddb1ca11a Mon Sep 17 00:00:00 2001 From: madmaxoft Date: Thu, 13 Feb 2014 12:48:22 +0100 Subject: MCADefrag: Fixed bugs, now produces valid MCA files. --- Tools/MCADefrag/MCADefrag.cpp | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) (limited to 'Tools/MCADefrag/MCADefrag.cpp') diff --git a/Tools/MCADefrag/MCADefrag.cpp b/Tools/MCADefrag/MCADefrag.cpp index 2f24469c9..59a67e45d 100644 --- a/Tools/MCADefrag/MCADefrag.cpp +++ b/Tools/MCADefrag/MCADefrag.cpp @@ -11,6 +11,13 @@ +// An array of 4096 zero bytes, used for writing the padding +static const Byte g_Zeroes[4096] = {0}; + + + + + int main(int argc, char ** argv) { new cMCLogger(Printf("Defrag_%08x.log", time(NULL))); @@ -275,7 +282,7 @@ bool cMCADefrag::cThread::WriteChunk(cFile & a_File, Byte * a_LocationRaw) a_LocationRaw[0] = m_CurrentSectorOut >> 16; a_LocationRaw[1] = (m_CurrentSectorOut >> 8) & 0xff; a_LocationRaw[2] = m_CurrentSectorOut & 0xff; - a_LocationRaw[3] = (m_CompressedChunkDataSize + (4 KiB) - 1) / (4 KiB); + a_LocationRaw[3] = (m_CompressedChunkDataSize + (4 KiB) + 3) / (4 KiB); // +3 because the m_CompressedChunkDataSize doesn't include the exact-length // Write the data length: Byte Buf[4]; @@ -295,6 +302,17 @@ bool cMCADefrag::cThread::WriteChunk(cFile & a_File, Byte * a_LocationRaw) LOGWARNING("Failed to write chunk data!"); return false; } + + // Pad onto the next sector: + int NumPadding = a_LocationRaw[3] * 4096 - (m_CompressedChunkDataSize + 4); + ASSERT(NumPadding >= 0); + if ((NumPadding > 0) && (a_File.Write(g_Zeroes, NumPadding) != NumPadding)) + { + LOGWARNING("Failed to write padding"); + return false; + } + + m_CurrentSectorOut += a_LocationRaw[3]; return true; } -- cgit v1.2.3 From 41ab8260f7b50bedbf06ea0effcb6147cb648685 Mon Sep 17 00:00:00 2001 From: madmaxoft Date: Thu, 13 Feb 2014 16:54:29 +0100 Subject: MCADefrag: Implemented recompression. This finalizes #639. --- Tools/MCADefrag/MCADefrag.cpp | 110 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 105 insertions(+), 5 deletions(-) (limited to 'Tools/MCADefrag/MCADefrag.cpp') diff --git a/Tools/MCADefrag/MCADefrag.cpp b/Tools/MCADefrag/MCADefrag.cpp index 59a67e45d..a2de7f957 100644 --- a/Tools/MCADefrag/MCADefrag.cpp +++ b/Tools/MCADefrag/MCADefrag.cpp @@ -6,6 +6,7 @@ #include "Globals.h" #include "MCADefrag.h" #include "MCLogger.h" +#include "zlib/zlib.h" @@ -40,7 +41,8 @@ int main(int argc, char ** argv) // cMCADefrag: cMCADefrag::cMCADefrag(void) : - m_NumThreads(1) + m_NumThreads(4), + m_ShouldRecompress(true) { } @@ -113,7 +115,8 @@ AString cMCADefrag::GetNextFileName(void) cMCADefrag::cThread::cThread(cMCADefrag & a_Parent) : super("MCADefrag thread"), - m_Parent(a_Parent) + m_Parent(a_Parent), + m_IsChunkUncompressed(false) { } @@ -204,6 +207,7 @@ void cMCADefrag::cThread::ProcessFile(const AString & a_FileName) // Chunk not present continue; } + m_IsChunkUncompressed = false; if (!ReadChunk(In, Locations + idx)) { LOGWARNING("Cannot read chunk #%d from file %s. Skipping file.", i, a_FileName.c_str()); @@ -266,7 +270,15 @@ bool cMCADefrag::cThread::ReadChunk(cFile & a_File, const Byte * a_LocationRaw) return false; } - // TODO: Uncompress the data if recompression is active + // Uncompress the data if recompression is active + if (m_Parent.m_ShouldRecompress) + { + m_IsChunkUncompressed = UncompressChunk(); + if (!m_IsChunkUncompressed) + { + LOGINFO("Chunk failed to uncompress, will be copied verbatim instead."); + } + } return true; } @@ -277,12 +289,21 @@ bool cMCADefrag::cThread::ReadChunk(cFile & a_File, const Byte * a_LocationRaw) bool cMCADefrag::cThread::WriteChunk(cFile & a_File, Byte * a_LocationRaw) { - // TODO: Recompress the data if recompression is active + // Recompress the data if recompression is active: + if (m_Parent.m_ShouldRecompress) + { + if (!CompressChunk()) + { + LOGINFO("Chunk failed to recompress, will be coped verbatim instead."); + } + } + // Update the Location: a_LocationRaw[0] = m_CurrentSectorOut >> 16; a_LocationRaw[1] = (m_CurrentSectorOut >> 8) & 0xff; a_LocationRaw[2] = m_CurrentSectorOut & 0xff; a_LocationRaw[3] = (m_CompressedChunkDataSize + (4 KiB) + 3) / (4 KiB); // +3 because the m_CompressedChunkDataSize doesn't include the exact-length + m_CurrentSectorOut += a_LocationRaw[3]; // Write the data length: Byte Buf[4]; @@ -312,7 +333,86 @@ bool cMCADefrag::cThread::WriteChunk(cFile & a_File, Byte * a_LocationRaw) return false; } - m_CurrentSectorOut += a_LocationRaw[3]; + return true; +} + + + + + +bool cMCADefrag::cThread::UncompressChunk(void) +{ + switch (m_CompressedChunkData[0]) + { + case COMPRESSION_GZIP: return UncompressChunkGzip(); + case COMPRESSION_ZLIB: return UncompressChunkZlib(); + } + LOGINFO("Chunk is compressed with in an unknown algorithm"); + return false; +} + + + + + +bool cMCADefrag::cThread::UncompressChunkGzip(void) +{ + // TODO + // This format is not used in practice + return false; +} + + + + + +bool cMCADefrag::cThread::UncompressChunkZlib(void) +{ + // Uncompress the data: + z_stream strm; + strm.zalloc = (alloc_func)NULL; + strm.zfree = (free_func)NULL; + strm.opaque = NULL; + inflateInit(&strm); + strm.next_out = m_RawChunkData; + strm.avail_out = sizeof(m_RawChunkData); + strm.next_in = m_CompressedChunkData + 1; // The first byte is the compression method, skip it + strm.avail_in = m_CompressedChunkDataSize; + int res = inflate(&strm, Z_FINISH); + inflateEnd(&strm); + if (res != Z_STREAM_END) + { + LOGWARNING("Failed to uncompress chunk data: %s", strm.msg); + return false; + } + m_RawChunkDataSize = strm.total_out; + + return true; +} + + + + + +bool cMCADefrag::cThread::CompressChunk(void) +{ + // Check that the compressed data can fit: + uLongf CompressedSize = compressBound(m_RawChunkDataSize); + if (CompressedSize > sizeof(m_CompressedChunkData)) + { + LOGINFO("Too much data for the internal compression buffer!"); + return false; + } + + // Compress the data using the highest compression factor: + int errorcode = compress2(m_CompressedChunkData + 1, &CompressedSize, m_RawChunkData, m_RawChunkDataSize, Z_BEST_COMPRESSION); + if (errorcode != Z_OK) + { + LOGINFO("Recompression failed: %d", errorcode); + return false; + } + m_CompressedChunkData[0] = COMPRESSION_ZLIB; + m_CompressedChunkDataSize = CompressedSize + 1; return true; } -- cgit v1.2.3