From 3bf2b0e63052bb5537398f38ecc5b19fec7dec39 Mon Sep 17 00:00:00 2001 From: bigbiff bigbiff Date: Mon, 21 Jan 2013 21:26:43 -0500 Subject: change tar create to pthread Change-Id: I5a33d207ec6683de20da37e6f4f174c67785fc52 --- gui/action.cpp | 3 +- partition.cpp | 26 +++++--- twrpTar.cpp | 186 +++++++++++++++++++++++++++++++++++++-------------------- twrpTar.hpp | 37 +++++++----- variables.h | 4 +- 5 files changed, 166 insertions(+), 90 deletions(-) diff --git a/gui/action.cpp b/gui/action.cpp index 14ef71609..997cf558e 100644 --- a/gui/action.cpp +++ b/gui/action.cpp @@ -249,10 +249,11 @@ int GUIAction::doActions() LOGE("Error setting pthread_attr_setscope\n"); return -1; } - if (pthread_attr_setstacksize(&tattr, 524288)) { + /*if (pthread_attr_setstacksize(&tattr, 524288)) { LOGE("Error setting pthread_attr_setstacksize\n"); return -1; } + */ LOGI("Creating thread\n"); int ret = pthread_create(&t, &tattr, thread_start, this); if (ret) { diff --git a/partition.cpp b/partition.cpp index d16ed7c74..c532fa830 100644 --- a/partition.cpp +++ b/partition.cpp @@ -1255,7 +1255,7 @@ bool TWPartition::Backup_Tar(string backup_folder) { unsigned long long total_bsize = 0, file_size; twrpTar tar; vector files; - + if (!Mount(true)) return false; @@ -1276,7 +1276,9 @@ bool TWPartition::Backup_Tar(string backup_folder) { // This backup needs to be split into multiple archives ui_print("Breaking backup file into multiple archives...\n"); sprintf(back_name, "%s", Backup_Path.c_str()); - backup_count = tar.Split_Archive(back_name, Full_FileName); + tar.setdir(back_name); + tar.setfn(Full_FileName); + backup_count = tar.splitArchiveThread(); if (backup_count == -1) { LOGE("Error tarring split files!\n"); return false; @@ -1285,14 +1287,18 @@ bool TWPartition::Backup_Tar(string backup_folder) { } else { Full_FileName = backup_folder + "/" + Backup_FileName; if (use_compression) { - if (tar.createTGZ(Backup_Path, Full_FileName) != 0) - return false; + tar.setdir(Backup_Path); + tar.setfn(Full_FileName); + if (tar.createTarGZThread() != 0) + return -1; string gzname = Full_FileName + ".gz"; rename(gzname.c_str(), Full_FileName.c_str()); } else { - if (tar.create(Backup_Path, Full_FileName) != 0) - return false; + tar.setdir(Backup_Path); + tar.setfn(Full_FileName); + if (tar.createTarThread() != 0) + return -1; } if (TWFunc::Get_File_Size(Full_FileName) == 0) { LOGE("Backup file size for '%s' is 0 bytes.\n", Full_FileName.c_str()); @@ -1381,7 +1387,9 @@ bool TWPartition::Restore_Tar(string restore_folder, string Restore_File_System) ui_print("Restoring archive %i...\n", index); LOGI("Restoring '%s'...\n", Full_FileName.c_str()); twrpTar tar; - if (tar.extract("/", Full_FileName) != 0) + tar.setdir("/"); + tar.setfn(Full_FileName); + if (tar.extractTarThread() != 0) return false; sprintf(split_index, "%03i", index); Full_FileName = restore_folder + "/" + Backup_FileName + split_index; @@ -1393,7 +1401,9 @@ bool TWPartition::Restore_Tar(string restore_folder, string Restore_File_System) } } else { twrpTar tar; - if (tar.extract(Backup_Path, Full_FileName) != 0) + tar.setdir(Backup_Path); + tar.setfn(Full_FileName); + if (tar.extractTarThread() != 0) return false; } return true; diff --git a/twrpTar.cpp b/twrpTar.cpp index 0008de457..78409c7e1 100644 --- a/twrpTar.cpp +++ b/twrpTar.cpp @@ -28,29 +28,79 @@ extern "C" { #include #include #include +#include #include #include #include "twrpTar.hpp" #include "common.h" #include "data.hpp" #include "variables.h" -#include #include "twrp-functions.hpp" using namespace std; -int twrpTar::Generate_Multiple_Archives(string Path, string fn) { +void twrpTar::setfn(string fn) { + tarfn = fn; +} + +void twrpTar::setdir(string dir) { + tardir = dir; +} + +int twrpTar::createTarGZThread() { + pthread_t thread; + ThreadPtr tarptr = &twrpTar::createTGZ; + PThreadPtr p = *(PThreadPtr*)&tarptr; + pthread_create(&thread, NULL, p, this); + if(pthread_join(thread, NULL)) { + return -1; + } + return 0; +} + +int twrpTar::createTarThread() { + pthread_t thread; + ThreadPtr tarptr = &twrpTar::create; + PThreadPtr p = *(PThreadPtr*)&tarptr; + pthread_create(&thread, NULL, p, this); + if(pthread_join(thread, NULL)) { + return -1; + } + return 0; +} + +int twrpTar::extractTarThread() { + pthread_t thread; + ThreadPtr tarptr = &twrpTar::extract; + PThreadPtr p = *(PThreadPtr*)&tarptr; + pthread_create(&thread, NULL, p, this); + if(pthread_join(thread, NULL)) { + return -1; + } + return 0; +} + +int twrpTar::splitArchiveThread() { + pthread_t thread; + ThreadPtr tarptr = &twrpTar::Split_Archive; + PThreadPtr p = *(PThreadPtr*)&tarptr; + pthread_create(&thread, NULL, p, this); + if(pthread_join(thread, NULL)) { + return -1; + } + return 0; +} + +int twrpTar::Generate_Multiple_Archives(string Path) { DIR* d; struct dirent* de; struct stat st; string FileName; char actual_filename[255]; - sprintf(actual_filename, fn.c_str(), Archive_File_Count); - if (has_data_media == 1 && Path.size() >= 11 && strncmp(Path.c_str(), "/data/media", 11) == 0) return 0; // Skip /data/media - LOGI("Path: '%s', archive filename: '%s'\n", Path.c_str(), actual_filename); + LOGI("Path: '%s', archive filename: '%s'\n", Path.c_str(), tarfn.c_str()); d = opendir(Path.c_str()); if (d == NULL) @@ -68,13 +118,16 @@ int twrpTar::Generate_Multiple_Archives(string Path, string fn) { if (de->d_type == DT_DIR && strcmp(de->d_name, ".") != 0 && strcmp(de->d_name, "..") != 0) { unsigned long long folder_size = TWFunc::Get_Folder_Size(FileName, false); + tardir = FileName; if (Archive_Current_Size + folder_size > MAX_ARCHIVE_SIZE) { - if (Generate_Multiple_Archives(FileName, fn) < 0) + LOGI("Calling Generate_Multiple_Archives\n"); + if (Generate_Multiple_Archives(FileName) < 0) return -1; } else { //FileName += "/"; LOGI("Adding folder '%s'\n", FileName.c_str()); - if (tarDirs(FileName, actual_filename, true) < 0) + tardir = FileName; + if (tarDirs(true) < 0) return -1; Archive_Current_Size += folder_size; } @@ -84,22 +137,25 @@ int twrpTar::Generate_Multiple_Archives(string Path, string fn) { stat(FileName.c_str(), &st); if (Archive_Current_Size != 0 && Archive_Current_Size + st.st_size > MAX_ARCHIVE_SIZE) { - LOGI("Closing tar '%s', ", actual_filename); - closeTar(actual_filename, false); - Archive_File_Count++; - if (TWFunc::Get_File_Size(actual_filename) == 0) { - LOGE("Backup file size for '%s' is 0 bytes.\n", actual_filename); - return false; + LOGI("Closing tar '%s', ", tarfn.c_str()); + closeTar(false); + if (TWFunc::Get_File_Size(tarfn) == 0) { + LOGE("Backup file size for '%s' is 0 bytes.\n", tarfn.c_str()); + return -1; } + Archive_File_Count++; if (Archive_File_Count > 999) { LOGE("Archive count is too large!\n"); return -1; } + string temp = basefn + "%03i"; + sprintf(actual_filename, temp.c_str(), Archive_File_Count); + tarfn = actual_filename; Archive_Current_Size = 0; - sprintf(actual_filename, fn.c_str(), Archive_File_Count); - LOGI("Creating tar '%s'\n", actual_filename); + LOGI("Creating tar '%s'\n", tarfn.c_str()); ui_print("Creating archive %i...\n", Archive_File_Count + 1); - createTar(Path, actual_filename); + if (createTar() != 0) + return -1; } LOGI("Adding file: '%s'... ", FileName.c_str()); if (addFile(FileName, true) < 0) @@ -114,34 +170,35 @@ int twrpTar::Generate_Multiple_Archives(string Path, string fn) { return 0; } -int twrpTar::Split_Archive(string Path, string fn) +int twrpTar::Split_Archive() { - string temp = fn + "%03i"; + string temp = tarfn + "%03i"; char actual_filename[255]; + basefn = tarfn; Archive_File_Count = 0; Archive_Current_Size = 0; sprintf(actual_filename, temp.c_str(), Archive_File_Count); - createTar(Path, actual_filename); + tarfn = actual_filename; + createTar(); DataManager::GetValue(TW_HAS_DATA_MEDIA, has_data_media); ui_print("Creating archive 1...\n"); - if (Generate_Multiple_Archives(Path, temp) < 0) { + if (Generate_Multiple_Archives(tardir) < 0) { LOGE("Error generating file list\n"); return -1; } - sprintf(actual_filename, temp.c_str(), Archive_File_Count); - closeTar(actual_filename, false); + closeTar(false); LOGI("Done, created %i archives.\n", (Archive_File_Count++)); return (Archive_File_Count); } -int twrpTar::extractTar(string rootdir, string fn) { - char* charRootDir = (char*) rootdir.c_str(); +int twrpTar::extractTar() { + char* charRootDir = (char*) tardir.c_str(); bool gzip = false; - if (openTar(rootdir, fn, gzip) == -1) + if (openTar(gzip) == -1) return -1; if (tar_extract_all(t, charRootDir) != 0) { - LOGE("Unable to extract tar archive '%s'\n", fn.c_str()); + LOGE("Unable to extract tar archive '%s'\n", tarfn.c_str()); return -1; } if (tar_close(t) != 0) { @@ -151,7 +208,7 @@ int twrpTar::extractTar(string rootdir, string fn) { return 0; } -int twrpTar::extract(string rootdir, string fn) { +int twrpTar::extract() { int len = 3; char header[len]; string::size_type i = 0; @@ -159,7 +216,7 @@ int twrpTar::extract(string rootdir, string fn) { int secondbyte = 0; int ret; ifstream f; - f.open(fn.c_str(), ios::in | ios::binary); + f.open(tarfn.c_str(), ios::in | ios::binary); f.get(header, len); firstbyte = header[i] & 0xff; secondbyte = header[++i] & 0xff; @@ -167,27 +224,27 @@ int twrpTar::extract(string rootdir, string fn) { if (firstbyte == 0x1f && secondbyte == 0x8b) { //if you return the extractTGZ function directly, stack crashes happen LOGI("Extracting gzipped tar\n"); - ret = extractTGZ(rootdir, fn); + ret = extractTGZ(); return ret; } else { LOGI("Extracting uncompressed tar\n"); - return extractTar(rootdir, fn); + return extractTar(); } } -int twrpTar::tarDirs(string dir, string fn, bool include_root) { +int twrpTar::tarDirs(bool include_root) { DIR* d; - string mainfolder = dir + "/", subfolder; + string mainfolder = tardir + "/", subfolder; char buf[1024]; - char* charTarFile = (char*) fn.c_str(); - d = opendir(dir.c_str()); + char* charTarFile = (char*) tarfn.c_str(); + d = opendir(tardir.c_str()); if (d != NULL) { struct dirent* de; while ((de = readdir(d)) != NULL) { LOGI("adding %s\n", de->d_name); #ifdef RECOVERY_SDCARD_ON_DATA - if ((dir == "/data" || dir == "/data/") && strcmp(de->d_name, "media") == 0) continue; + if ((tardir == "/data" || tardir == "/data/") && strcmp(de->d_name, "media") == 0) continue; #endif if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue; @@ -195,7 +252,7 @@ int twrpTar::tarDirs(string dir, string fn, bool include_root) { subfolder += de->d_name; strcpy(buf, subfolder.c_str()); if (de->d_type == DT_DIR) { - if (include_root) { + if (include_root) { if (tar_append_tree(t, buf, NULL) != 0) { LOGE("Error appending '%s' to tar archive '%s'\n", buf, charTarFile); return -1; @@ -208,7 +265,7 @@ int twrpTar::tarDirs(string dir, string fn, bool include_root) { return -1; } } - } else if (dir != "/" && (de->d_type == DT_REG || de->d_type == DT_LNK)) { + } else if (tardir != "/" && (de->d_type == DT_REG || de->d_type == DT_LNK)) { if (addFile(buf, include_root) != 0) return -1; } @@ -219,24 +276,24 @@ int twrpTar::tarDirs(string dir, string fn, bool include_root) { return 0; } -int twrpTar::createTGZ(string dir, string fn) { +int twrpTar::createTGZ() { bool gzip = true; - if (createTar(dir, fn) == -1) + if (createTar() == -1) return -1; - if (tarDirs(dir, fn, false) == -1) + if (tarDirs(false) == -1) return -1; - if (closeTar(fn, gzip) == -1) + if (closeTar(gzip) == -1) return -1; return 0; } -int twrpTar::create(string dir, string fn) { +int twrpTar::create() { bool gzip = false; - if (createTar(dir, fn) == -1) + if (createTar() == -1) return -1; - if (tarDirs(dir, fn, false) == -1) + if (tarDirs(false) == -1) return -1; - if (closeTar(fn, gzip) == -1) + if (closeTar(gzip) == -1) return -1; return 0; } @@ -250,7 +307,7 @@ int twrpTar::addFilesToExistingTar(vector files, string fn) { if (tar_open(&t, charTarFile, NULL, O_WRONLY | O_APPEND | O_LARGEFILE, 0644, TAR_GNU) == -1) return -1; for (unsigned int i = 0; i < files.size(); ++i) { - char* file = (char*) files.at(i).c_str(); + char* file = (char*) files.at(i).c_str(); if (tar_append_file(t, file, file) == -1) return -1; } @@ -261,15 +318,14 @@ int twrpTar::addFilesToExistingTar(vector files, string fn) { return 0; } -int twrpTar::createTar(string rootdir, string fn) { - char* charTarFile = (char*) fn.c_str(); - char* charRootDir = (char*) rootdir.c_str(); +int twrpTar::createTar() { + char* charTarFile = (char*) tarfn.c_str(); + char* charRootDir = (char*) tardir.c_str(); int use_compression = 0; DataManager::GetValue(TW_USE_COMPRESSION_VAR, use_compression); - LOGI("2nd compression\n"); if (use_compression) { - string cmd = "pigz - > '" + fn + "'"; + string cmd = "pigz - > '" + tarfn + "'"; p = popen(cmd.c_str(), "w"); fd = fileno(p); if (!p) return -1; @@ -277,7 +333,7 @@ int twrpTar::createTar(string rootdir, string fn) { pclose(p); return -1; } - } + } else { if (tar_open(&t, charTarFile, NULL, O_WRONLY | O_CREAT | O_LARGEFILE, 0644, TAR_GNU) == -1) return -1; @@ -285,13 +341,13 @@ int twrpTar::createTar(string rootdir, string fn) { return 0; } -int twrpTar::openTar(string rootdir, string fn, bool gzip) { - char* charRootDir = (char*) rootdir.c_str(); - char* charTarFile = (char*) fn.c_str(); +int twrpTar::openTar(bool gzip) { + char* charRootDir = (char*) tardir.c_str(); + char* charTarFile = (char*) tarfn.c_str(); if (gzip) { LOGI("Opening as a gzip\n"); - string cmd = "pigz -d -c '" + fn + "'"; + string cmd = "pigz -d -c '" + tarfn + "'"; FILE* pipe = popen(cmd.c_str(), "r"); int fd = fileno(pipe); if (!pipe) return -1; @@ -344,7 +400,7 @@ int twrpTar::addFile(string fn, bool include_root) { return 0; } -int twrpTar::closeTar(string fn, bool gzip) { +int twrpTar::closeTar(bool gzip) { int use_compression; DataManager::GetValue(TW_USE_COMPRESSION_VAR, use_compression); @@ -354,7 +410,7 @@ int twrpTar::closeTar(string fn, bool gzip) { return -1; } if (tar_close(t) != 0) { - LOGE("Unable to close tar archive: '%s'\n", fn.c_str()); + LOGE("Unable to close tar archive: '%s'\n", tarfn.c_str()); return -1; } if (use_compression || gzip) { @@ -369,19 +425,19 @@ int twrpTar::removeEOT(string tarFile) { char* charTarFile = (char*) tarFile.c_str(); off_t tarFileEnd; while (th_read(t) == 0) { - if (TH_ISREG(t)) + if (TH_ISREG(t)) tar_skip_regfile(t); tarFileEnd = lseek(t->fd, 0, SEEK_CUR); - } + } if (tar_close(t) == -1) return -1; - if (truncate(charTarFile, tarFileEnd) == -1) + if (truncate(charTarFile, tarFileEnd) == -1) return -1; return 0; } int twrpTar::compress(string fn) { - string cmd = "pigz " + fn; + string cmd = "pigz " + fn; p = popen(cmd.c_str(), "r"); if (!p) return -1; char buffer[128]; @@ -394,11 +450,11 @@ int twrpTar::compress(string fn) { return 0; } -int twrpTar::extractTGZ(string rootdir, string fn) { - string splatrootdir(rootdir); +int twrpTar::extractTGZ() { + string splatrootdir(tardir); bool gzip = true; char* splatCharRootDir = (char*) splatrootdir.c_str(); - if (openTar(rootdir, fn, gzip) == -1) + if (openTar(gzip) == -1) return -1; int ret = tar_extract_all(t, splatCharRootDir); if (tar_close(t) != 0) { diff --git a/twrpTar.hpp b/twrpTar.hpp index 554a01dbd..db9cf9bb8 100644 --- a/twrpTar.hpp +++ b/twrpTar.hpp @@ -32,30 +32,39 @@ using namespace std; class twrpTar { public: - int create(string dir, string fn); - int createTGZ(string dir, string fn); - int extract(string rootDir, string fn); + int extract(); int compress(string fn); - int extractTGZ(string rootdir, string fn); int uncompress(string fn); int addFilesToExistingTar(vector files, string tarFile); - int createTar(string dir, string fn); - int openTar(string rootdir, string fn, bool gzip); + int createTar(); int addFile(string fn, bool include_root); - int closeTar(string fn, bool gzip); - int Split_Archive(string Path, string fn); + int closeTar(bool gzip); + int createTarGZThread(); + int createTarThread(); + int extractTarThread(); + int splitArchiveThread(); + void setfn(string fn); + void setdir(string dir); private: + int createTGZ(); + int create(); + int Split_Archive(); int removeEOT(string tarFile); - int extractTar(string rootdir, string fn); - int tarDirs(string dir, string fn, bool include_root); - int Generate_Multiple_Archives(string Path, string fn); - - private: + int extractTar(); + int tarDirs(bool include_root); + int Generate_Multiple_Archives(string Path); + string Strip_Root_Dir(string Path); + int extractTGZ(); + int openTar(bool gzip); int has_data_media; int Archive_File_Count; unsigned long long Archive_Current_Size; - string Strip_Root_Dir(string Path); TAR *t; FILE* p; int fd; + string tardir; + string tarfn; + string basefn; + typedef int (twrpTar::*ThreadPtr)(void); + typedef void* (*PThreadPtr)(void*); }; diff --git a/variables.h b/variables.h index f56a46943..91c1f2c13 100644 --- a/variables.h +++ b/variables.h @@ -164,7 +164,7 @@ // tw_sp2_is_mountable // tw_sp3_is_mountable -// Max archive size for tar backups before we split (4GB) -#define MAX_ARCHIVE_SIZE 4294967296LLU +// Max archive size for tar backups before we split (1.5GB) +#define MAX_ARCHIVE_SIZE 1610612736LLU #endif // _VARIABLES_HEADER_ -- cgit v1.2.3