From a2bfba4ad37714cdb4a0b7f5e5e30a6a05e5df8d Mon Sep 17 00:00:00 2001 From: Ben Chamberland <85132405+chxmberland@users.noreply.github.com> Date: Tue, 11 Jun 2024 11:16:33 -0700 Subject: [PATCH] Add .rvpkg bundle support for rvpkg command line tools and UI (#471) ### Linked issues ### Summarize your change. Users can now upload multiple packages at the same time through a compressed zip file using the extension **.rvpkgs**. These **.rvpkgs** files can be uploaded using the `rvpkg -add ...` command in the shell, or directly in the UI under the `Preferences/Packages` section of the menu. In either case, **.rvpkgs** files will be immediately unzipped into the packages they contain before each package is added to OpenRV. Users can also add and install these **.rvpkgs** bundles using a chain command as follows. `rvpkg -force -install -add ` ### Describe the reason for the change. ### Describe what you have tested and on which operating system. This change was tested on MacOS. To replicate the test, recreate the following steps. **Creating an .rvpkgs file** 1. Find the .rvpkg files you want to bundle 2. Create a zip folder containing these .rvpkg files 3. Rename this zip file following `-.rvpkgs` **Testing UI upload** 1. Launch OpenRV 2. Navigate to the `Preferences/Packages` section of the menu 3. Select `Add Packages...` 4. Select your own **.rvpkgs** file 5. Click open 6. The packages zipped into the **.rvpkgs** file should be added directly into RV **Testing Command Line** 1. Determine the location of your **.rvpkgs** file 2. Determine the location you want to add/install your packages to 3. Execute either of the following commands `rvpkg -force -install -add ` `rvpkg -add ` Note that packages are often installed to `~/Library/Application\ Support/RV/` on Mac. ### Add a list of changes, and note any that might need special attention during the review. - [ ] Added bundle detection and handling functionality to the rvpkg main file (specifically -add and -install sections) - [ ] Minor bug fix in RvPreferences where text indicated incorrectly that packages could not be uninstalled - [ ] Added a multitude of utility functions to the PackageManager, compartmentalizing and globalizing unzip and bundle detection functionality Signed-off-by: Ben Chamberland --- src/bin/apps/rvpkg/main.cpp | 79 ++++++-- src/lib/app/RvCommon/RvPreferences.cpp | 10 +- src/lib/app/RvPackage/PackageManager.cpp | 186 ++++++++++++++++-- .../app/RvPackage/RvPackage/PackageManager.h | 7 +- 4 files changed, 244 insertions(+), 38 deletions(-) diff --git a/src/bin/apps/rvpkg/main.cpp b/src/bin/apps/rvpkg/main.cpp index b5aafd64..3bc3d469 100644 --- a/src/bin/apps/rvpkg/main.cpp +++ b/src/bin/apps/rvpkg/main.cpp @@ -298,18 +298,18 @@ utf8Main(int argc, char *argv[]) if (inputArgs.empty() && !list && !info && !env) { - cout << "ERROR: use -help for usage" << endl; + cerr << "ERROR: use -help for usage" << endl; } if (add && remove) { - cout << "ERROR: only one of -add or -remove allowed" << endl; + cerr << "ERROR: only one of -add or -remove allowed" << endl; exit(-1); } if (onlyDir && (withDir || addDir)) { - cout << "ERROR: -only cannot be used with -include or -add" << endl; + cerr << "ERROR: -only cannot be used with -include or -add" << endl; exit(-1); } @@ -320,25 +320,25 @@ utf8Main(int argc, char *argv[]) if ((remove && install) || (add && uninstall)) { - cout << "ERROR: conflicting arguments" << endl; + cerr << "ERROR: conflicting arguments" << endl; exit(-1); } if ((remove || add || install || uninstall) && (list || info)) { - cout << "ERROR: -list and -info do not work with other arguments" << endl; + cerr << "ERROR: -list and -info do not work with other arguments" << endl; exit(-1); } if (list && info) { - cout << "ERROR: only one of -list or -info allowed" << endl; + cerr << "ERROR: only one of -list or -info allowed" << endl; exit(-1); } if ((info || install || uninstall || remove || add) && inputArgs.empty()) { - cout << "ERROR: need more arguments" << endl; + cerr << "ERROR: need more arguments" << endl; exit(-1); } @@ -489,7 +489,7 @@ utf8Main(int argc, char *argv[]) if (!p.optional) { - cout << "ERROR: " << p.file.toUtf8().constData() + cerr << "ERROR: " << p.file.toUtf8().constData() << " is not an optional package -- ignoring" << endl; continue; @@ -500,7 +500,7 @@ utf8Main(int argc, char *argv[]) if (!info.exists()) { - cout << "ERROR: " << p.file.toUtf8().constData() << " doesn't exist -- ignoring" << endl; + cerr << "ERROR: " << p.file.toUtf8().constData() << " doesn't exist -- ignoring" << endl; continue; } @@ -512,7 +512,7 @@ utf8Main(int argc, char *argv[]) if (!rvloadInfo.exists()) { - cout << "ERROR: missing rvload2 file at " + cerr << "ERROR: missing rvload2 file at " << rvload2.toUtf8().constData() << " -- ignoring" << endl; @@ -551,9 +551,49 @@ utf8Main(int argc, char *argv[]) { QStringList files; + // Will hold added packages from a bundle if an bundle is passes + std::vector addedPackages = {}; + for (size_t i = 0; i < inputArgs.size(); i++) { - files.push_back(inputArgs[i].c_str()); + string curFile = inputArgs[i]; + + // Checking to see if the file is a bundle + if (manager.isBundle(QString::fromStdString(curFile))) { + cout << "INFO: Bundle detected, unpacking now." << endl; + + // Unpacking bundle + QString toUnzip = QString::fromStdString(curFile); + string addDirStr = addDir; + QString outputDir = QString::fromStdString(addDirStr.append("/Packages")); + addedPackages = manager.handleBundle(toUnzip, outputDir); + + // Ensuring that the bundle unpacked successfully + if (addedPackages.size() > 0) + { + cout << "INFO: Bundle unpacked successfully " << endl; + cout << "INFO: Added the following packages..." << endl; + + for (QString s : addedPackages) + { + cout << s.toStdString() << endl; + } + } else { + cerr << "ERROR: Unable to install bundle." << endl; + } + } else { + files.push_back(inputArgs[i].c_str()); + } + } + + // Adding bundle subpackages to input arguments to allow for chain calls (e.g. -install -add ...) + if (addedPackages.size() > 0) + { + + for (QString s : addedPackages) + { + inputArgs.push_back(s.toStdString()); + } } if (!manager.addPackages(files, addDir)) @@ -571,19 +611,24 @@ utf8Main(int argc, char *argv[]) { QString name(inputArgs[i].c_str()); QString base = name.split("/").back(); + + // Bundles do not need to be modified and should instead be removed + if (manager.isBundle(base)) continue; + QDir dir(addDir); - dir.cd("Packages"); + dir.cd("Packages"); // Automatically navigating into the Packages directory if (!dir.exists()) exit(-1); QString path = dir.absoluteFilePath(base); inputArgs[i] = path.toUtf8().constData(); - cout << inputArgs[i] << endl; } } if (install) { + cout << "INFO: Installing package..." << endl; + vector indices; matchingPackages(packages, indices); @@ -596,17 +641,19 @@ utf8Main(int argc, char *argv[]) { PackageManager::Package& p = packages[indices[i]]; + // Ensuring package not installed already if (p.installed) { cout << "WARNING: " << p.name << " is already installed" << endl; - } + } + else { cout << "INFO: installing " << p.file << endl; if (!manager.installPackage(p)) { - cout << "ERROR: failed to install " << p.file << endl; + cerr << "ERROR: failed to install " << p.file << endl; } } } @@ -658,4 +705,4 @@ utf8Main(int argc, char *argv[]) } return 0; -} +} \ No newline at end of file diff --git a/src/lib/app/RvCommon/RvPreferences.cpp b/src/lib/app/RvCommon/RvPreferences.cpp index 66d20c70..a7e4dec7 100644 --- a/src/lib/app/RvCommon/RvPreferences.cpp +++ b/src/lib/app/RvCommon/RvPreferences.cpp @@ -2937,7 +2937,7 @@ RvPreferences::addPackage(bool) QFileDialog* fileDialog = new QFileDialog(this, "Select rvpkg RV package files", dirname, - "rvpkg Package Files (*.zip *.rvpkg)"); + "rvpkg Package Files (*.zip *.rvpkg *.rvpkgs)"); QStringList files; if (fileDialog->exec()) files = fileDialog->selectedFiles(); @@ -2961,7 +2961,7 @@ RvPreferences::addPackage(bool) QStringList files = QFileDialog::getOpenFileNames(this, "Select rvpkg RV package files", dirname, - "rvpkg Package Files (*.zip *.rvpkg)", + "rvpkg Package Files (*.zip *.rvpkg *.rvpkgs, *.rvpkgs)", &selectedFilter, options); #endif @@ -3119,7 +3119,7 @@ RvPreferences::installDependantPackages(const QString& msg) { QMessageBox box(this); box.setWindowTitle(tr("Some Packages Depend on This One")); - box.setText(tr("Can't uninstall package because some other packages dependend on this one. Try and uninstall them first?\n\nDetails:\n") + msg); + box.setText(tr("Can't install package because some other packages dependend on this one. Try and install them first?\n\nDetails:\n") + msg); box.setWindowModality(Qt::WindowModal); QPushButton* b1 = box.addButton(tr("Abort"), QMessageBox::RejectRole); @@ -4025,6 +4025,4 @@ RvPreferences::formatProfileChanged(int index) } -} // Rv - - +} // Rv \ No newline at end of file diff --git a/src/lib/app/RvPackage/PackageManager.cpp b/src/lib/app/RvPackage/PackageManager.cpp index f82e81d9..d24c018f 100644 --- a/src/lib/app/RvPackage/PackageManager.cpp +++ b/src/lib/app/RvPackage/PackageManager.cpp @@ -16,11 +16,143 @@ #include #include #include +#include namespace Rv { using namespace std; + QString extractFile(unzFile uf, const string& outputPath) { + char filename[256]; + unz_file_info fileInfo; + + if (unzGetCurrentFileInfo(uf, &fileInfo, filename, sizeof(filename), nullptr, 0, nullptr, 0) != UNZ_OK) { + cerr << "ERROR: Unable to get file info" << endl; + return ""; + } + + if (unzOpenCurrentFile(uf) != UNZ_OK) { + cerr << "ERROR: Unable to open file in zip" << endl; + return ""; + } + + string fullPath = outputPath + "/" + filename; + FILE* outFile = std::fopen(fullPath.c_str(), "wb"); + if (outFile == nullptr) { + cerr << "ERROR: Unable to open output file" << endl; + unzCloseCurrentFile(uf); + return ""; + } + + char buffer[8192]; + int bytesRead = 0; + while ((bytesRead = unzReadCurrentFile(uf, buffer, sizeof(buffer))) > 0) { + fwrite(buffer, 1, bytesRead, outFile); + } + + fclose(outFile); + unzCloseCurrentFile(uf); + + return QString::fromStdString(fullPath); + } + + vector unzipBundle(const string& zipFilePath, const string& outputPath) + { + vector includedPackages; + + // Opening zip file + unzFile uf = unzOpen(zipFilePath.c_str()); + if (uf == nullptr) { + cerr << "ERROR: Unable to open zip file" << endl; + return {}; + } + + // Checking for the presence of files + if (unzGoToFirstFile(uf) != UNZ_OK) { + cerr << "ERROR: Unable to find first file in zip" << endl; + unzClose(uf); + return {}; + } + + // Extracting each file in the zip + do { + QString extractedFilePath = extractFile(uf, outputPath); + if (extractedFilePath == "") { + cerr << "ERROR: Unable to extract zip file" << endl; + } else { + includedPackages.push_back(extractedFilePath); + } + } while (unzGoToNextFile(uf) == UNZ_OK); + + unzClose(uf); + + return includedPackages; + } + + bool PackageManager::isBundle(const QString& infileNonCanonical) + { + + // Getting package file extension + string filePath = infileNonCanonical.toStdString(); + size_t dot = filePath.rfind('.'); + if (dot == string::npos) { + cerr << "ERROR: Unsupported file type. Ensure each package has the extension 'rvpkg', 'zip' or 'rvpkgs'" << endl; + return false; + } + + // Edge case, last character is dot + if (dot == filePath.size() - 1) { + cerr << "ERROR: File paths cannot end with a dot/period. Extension cannot be determined." << endl; + return false; + } + + // File extension + string ext = filePath.substr(dot + 1); + if (ext == "rvpkgs") { + return true; + + // May or may not be a bundle + } else if (ext == "zip") { + + unzFile uf = unzOpen(filePath.c_str()); + if (!uf) { + cerr << "ERROR: Failed to open zip file" << endl; + return false; + } + + // Checking for presence of PACKAGE file + if (unzGoToFirstFile(uf) == UNZ_OK) { + do { + unz_file_info fileInfo; + char fileName[256]; + if ( + unzGetCurrentFileInfo(uf, &fileInfo, fileName, sizeof(fileName), nullptr, 0, nullptr, 0) == UNZ_OK + && strcmp(fileName, "PACKAGE") == 0 + ) { + return false; + } + } while (unzGoToNextFile(uf) == UNZ_OK); + } + + unzClose(uf); + return true; // Zip does not contain a PACKAGE file, therefore assume it is a bundle + } + + return false; // Extension is not zip or rvpkgs + } + + vector PackageManager::handleBundle(const QString& bundlePath, const QString& outputPath) { + + // Attempting to unzip bundle into the Packages directory + vector includedPackages = unzipBundle(bundlePath.toStdString(), outputPath.toStdString()); + if (includedPackages.size() == 0) { + cerr << "ERROR: Unable to unzip bundle." << endl; + return {}; + } + + return includedPackages; + } + bool PackageManager::m_ignorePrefs = false; PackageManager::PackageManager() : m_force( false ) {} @@ -400,7 +532,7 @@ namespace Rv if( unzLocateFile( file, filename.toUtf8().data(), 1 ) != UNZ_OK ) { - cout << "ERROR: reading zip file " << package.file.toUtf8().data() + cerr << "ERROR: reading zip file " << package.file.toUtf8().data() << endl; break; } @@ -420,7 +552,7 @@ namespace Rv bool success = outdir.mkpath( "." ); if( !success ) { - cout << "ERROR: Failed to create needed auxiliary directory: " + + cerr << "ERROR: Failed to create needed auxiliary directory: " + outdir.absolutePath().toStdString() << endl; } @@ -511,7 +643,7 @@ namespace Rv if( system( cmd.c_str() ) == -1 ) { - cout << "ERROR: executing command " << cmd << endl; + cerr << "ERROR: executing command " << cmd << endl; } // QProcess process; @@ -674,6 +806,7 @@ namespace Rv QFileInfo pfile( package.file ); QRegExp rvpkgRE( "(.*)-[0-9]+\\.[0-9]+\\.rvpkg" ); QRegExp zipRE( "(.*)\\.zip" ); + QRegExp rvpkgsRE( "(.*)-[0-9]+\\.[0-9]+\\.rvpkgs" ); QString pname = package.baseName; if( !supportdir.exists( pname ) ) supportdir.mkdir( pname ); @@ -894,8 +1027,10 @@ namespace Rv return true; } + // Called on every package on Preference menu load and everytime a new package or bundle is added void PackageManager::loadPackageInfo( const QString& infileNonCanonical ) { + if( findPackageIndexByZip( infileNonCanonical ) == -1 ) { // @@ -977,7 +1112,7 @@ namespace Rv if( !yaml_parser_initialize( &parser ) ) { - cout << "ERROR: Could not initialize the YAML parser object" + cerr << "ERROR: Could not initialize the YAML parser object" << endl; return; } @@ -1003,7 +1138,7 @@ namespace Rv { if( !yaml_parser_parse( &parser, &input_event ) ) { - cout << "ERROR: YAML parser failed on PACKAGE file in " + cerr << "ERROR: YAML parser failed on PACKAGE file in " << infile.toStdString() << endl; break; } @@ -1500,7 +1635,8 @@ namespace Rv for( size_t q = 0; q < entries.size(); q++ ) { if( entries[q].fileName().endsWith( ".zip" ) || - entries[q].fileName().endsWith( "rvpkg" ) ) + entries[q].fileName().endsWith( "rvpkg" ) || + entries[q].fileName().endsWith( "rvpkgs" )) { loadPackageInfo( entries[q].filePath() ); } @@ -1583,10 +1719,27 @@ namespace Rv // QRegExp rvpkgRE( "(.*)-[0-9]+\\.[0-9]+\\.rvpkg" ); + QRegExp rvpkgsRE( "(.*)-[0-9]+\\.[0-9]+\\.rvpkgs" ); // Bundle QRegExp zipRE( "(.*)\\.zip" ); for( size_t i = 0; i < files.size(); i++ ) { + + // Unpacking any bundles + if (isBundle(files[i])) { + cout << "INFO: Bundle detected, adding subpackages now." << endl; + + // Installing bundle + vector includedPackages = handleBundle(files[i], path); + if (includedPackages.size() > 0) { + cout << "INFO: Bundle subpackages added." << endl; + } else { + cerr << "ERROR: Unable to add subpackages." << endl; + } + + continue; // Bundle itself does not need to be copied + }; + QFileInfo info( files[i] ); if( !info.exists() ) @@ -1598,11 +1751,12 @@ namespace Rv } if( !rvpkgRE.exactMatch( info.fileName() ) && - !zipRE.exactMatch( info.fileName() ) ) + !zipRE.exactMatch( info.fileName() ) && + !rvpkgsRE.exactMatch(info.fileName() ) ) { QString t( "ERROR: Illegal package file name: " ); t += info.fileName(); - t += ", should be either .zip or \n-.rvpkg"; + t += ", should be either .zip, \n-.rvpkg or -.rvpkgs"; informPackageFailedToCopy( t ); return false; } @@ -1620,11 +1774,15 @@ namespace Rv } makeSupportDirTree( dir ); - dir.cd( "Packages" ); + dir.cd( "Packages" ); // User should not include the Packages directory in their add directory argument QStringList nocopy; for( size_t i = 0; i < files.size(); i++ ) { + + // Bundles do not need to be copied + if (isBundle(files[i])) continue; + QFileInfo info( files[i] ); QString fromFile( files[i] ); QString toFile( dir.absoluteFilePath( info.fileName() ) ); @@ -1770,9 +1928,9 @@ namespace Rv bool PackageManager::installDependantPackages( const QString& msg ) { return yesOrNo( "Some Packages Depend on This One", - "Can't uninstall package because some other packages " + "Can't install package because some other packages " "dependend on this one.", - msg, "Try and uninstall others first?" ); + msg, "Try and install others first?" ); } bool PackageManager::overwriteExistingFiles( const QString& msg ) @@ -1784,7 +1942,7 @@ namespace Rv void PackageManager::errorMissingPackageDependancies( const QString& msg ) { - cout << "ERROR: Some package dependancies are missing" << endl + cerr << "ERROR: Some package dependancies are missing" << endl << msg.toUtf8().constData() << endl; } @@ -1804,7 +1962,7 @@ namespace Rv void PackageManager::errorModeFileWriteFailed( const QString& file ) { - cout << "ERROR: File write failed: " << file.toUtf8().constData() << endl; + cerr << "ERROR: File write failed: " << file.toUtf8().constData() << endl; } void PackageManager::informPackageFailedToCopy( const QString& msg ) @@ -2180,4 +2338,4 @@ namespace Rv return false; } -} // namespace Rv +} // namespace Rv \ No newline at end of file diff --git a/src/lib/app/RvPackage/RvPackage/PackageManager.h b/src/lib/app/RvPackage/RvPackage/PackageManager.h index 4888d023..73071529 100644 --- a/src/lib/app/RvPackage/RvPackage/PackageManager.h +++ b/src/lib/app/RvPackage/RvPackage/PackageManager.h @@ -174,6 +174,9 @@ class PackageManager virtual bool installPackage(Package&); virtual bool uninstallPackage(Package&); + virtual bool isBundle(const QString&); + virtual std::vector handleBundle(const QString&, const QString&); + virtual ModeEntryList loadModeFile(const QString&); virtual void writeModeFile(const QString&, const ModeEntryList&, int version=0); @@ -182,7 +185,7 @@ class PackageManager virtual bool allowLoading(Package&,bool,int d=0); virtual bool makeSupportDirTree(QDir& root); - + // // Override these for UI versions. The defaults use cin/cout to // collect responses from the user. @@ -231,4 +234,4 @@ class PackageManager } // Rv -#endif // __RvPackage__PackageManager__h__ +#endif // __RvPackage__PackageManager__h__ \ No newline at end of file