From a023657348051cbc7c8ea29e3b37f3e2e3fd16d8 Mon Sep 17 00:00:00 2001 From: oldmud0 Date: Sat, 29 May 2021 10:43:53 -0500 Subject: Partial: Add mount list to options dialog --- src/path_functions.cpp | 2 -- 1 file changed, 2 deletions(-) (limited to 'src/path_functions.cpp') diff --git a/src/path_functions.cpp b/src/path_functions.cpp index 5107349..9700529 100644 --- a/src/path_functions.cpp +++ b/src/path_functions.cpp @@ -39,8 +39,6 @@ QString AOApplication::get_base_path() return base_path; } -QString AOApplication::get_data_path() { return get_base_path() + "data/"; } - QString AOApplication::get_theme_path(QString p_file, QString p_theme) { if (p_theme == "") -- cgit From d27501313cae78b838c1e738ebfaeae4740a23b4 Mon Sep 17 00:00:00 2001 From: oldmud0 Date: Sat, 5 Jun 2021 14:58:40 -0500 Subject: Finish mounting feature To pull this one off, a new class called VPath was created that denotes "virtual" paths that can exist anywhere among the list of mount points. It is functionally identical to QString, except that implicit conversion between QString and VPath is not allowed. This makes it easy to spot errors in path resolution at compile time, since get_real_path must be called to resolve a VPath into an absolute path that can be passed into a Qt function as a QString. Other functions, such as the get_*_suffix functions, also return an absolute path QString for convenience. As for the rest of the functions that return a VPath, you will need to call get_real_path yourself. Finally, a path resolution cache was added to try to avoid blowing up what is already a massive lookup cost for assets. The cache is invalidated when the mount path list is changed. Currently, this cache isn't bounded. Might need to write an LRU cache if issues arise. --- src/path_functions.cpp | 151 ++++++++++++++++++++++++++----------------------- 1 file changed, 81 insertions(+), 70 deletions(-) (limited to 'src/path_functions.cpp') diff --git a/src/path_functions.cpp b/src/path_functions.cpp index 9700529..b1a5e48 100644 --- a/src/path_functions.cpp +++ b/src/path_functions.cpp @@ -39,71 +39,58 @@ QString AOApplication::get_base_path() return base_path; } -QString AOApplication::get_theme_path(QString p_file, QString p_theme) +VPath AOApplication::get_theme_path(QString p_file, QString p_theme) { if (p_theme == "") p_theme = current_theme; - QString path = get_base_path() + "themes/" + p_theme + "/" + p_file; - return get_case_sensitive_path(path); + return VPath("themes/" + p_theme + "/" + p_file); } -QString AOApplication::get_character_path(QString p_char, QString p_file) +VPath AOApplication::get_character_path(QString p_char, QString p_file) { - QString path = get_base_path() + "characters/" + p_char + "/" + p_file; - return get_case_sensitive_path(path); + return VPath("characters/" + p_char + "/" + p_file); } -QString AOApplication::get_misc_path(QString p_misc, QString p_file) +VPath AOApplication::get_misc_path(QString p_misc, QString p_file) { - QString path = get_base_path() + "misc/" + p_misc + "/" + p_file; -#ifndef CASE_SENSITIVE_FILESYSTEM - return path; -#else - return get_case_sensitive_path(path); -#endif + return VPath("misc/" + p_misc + "/" + p_file); } -QString AOApplication::get_sounds_path(QString p_file) +VPath AOApplication::get_sounds_path(QString p_file) { - QString path = get_base_path() + "sounds/general/" + p_file; - return get_case_sensitive_path(path); + return VPath("sounds/general/" + p_file); } -QString AOApplication::get_music_path(QString p_song) +VPath AOApplication::get_music_path(QString p_song) { if (p_song.startsWith("http")) { - return p_song; // url + return VPath(p_song); // url } - QString path = get_base_path() + "sounds/music/" + p_song; - return get_case_sensitive_path(path); + return VPath("sounds/music/" + p_song); } -QString AOApplication::get_background_path(QString p_file) +VPath AOApplication::get_background_path(QString p_file) { - QString path = get_base_path() + "background/" + - w_courtroom->get_current_background() + "/" + p_file; if (courtroom_constructed) { - return get_case_sensitive_path(path); + return VPath("background/" + w_courtroom->get_current_background() + "/" + p_file); } return get_default_background_path(p_file); } -QString AOApplication::get_default_background_path(QString p_file) +VPath AOApplication::get_default_background_path(QString p_file) { - QString path = get_base_path() + "background/default/" + p_file; - return get_case_sensitive_path(path); + return VPath("background/default/" + p_file); } -QString AOApplication::get_evidence_path(QString p_file) +VPath AOApplication::get_evidence_path(QString p_file) { - QString path = get_base_path() + "evidence/" + p_file; - return get_case_sensitive_path(path); + return VPath("evidence/" + p_file); } -QStringList AOApplication::get_asset_paths(QString p_element, QString p_theme, QString p_subtheme, QString p_default_theme, QString p_misc, QString p_character, QString p_placeholder) +QVector AOApplication::get_asset_paths(QString p_element, QString p_theme, QString p_subtheme, QString p_default_theme, QString p_misc, QString p_character, QString p_placeholder) { - QStringList pathlist; - pathlist += p_element; // The path by itself + QVector pathlist; + pathlist += VPath(p_element); // The path by itself if (p_character != "") pathlist += get_character_path(p_character, p_element); // Character folder if (p_misc != "" && p_theme != "" && p_subtheme != "") @@ -125,52 +112,47 @@ QStringList AOApplication::get_asset_paths(QString p_element, QString p_theme, Q return pathlist; } -QString AOApplication::get_asset_path(QStringList pathlist) +QString AOApplication::get_asset_path(QVector pathlist) { - QString path; - for (QString p : pathlist) { - p = get_case_sensitive_path(p); - if (file_exists(p)) { - path = p; - break; + for (const VPath &p : pathlist) { + QString path = get_real_path(p); + if (!path.isEmpty()) { + return path; } } - return path; + return QString(); } -QString AOApplication::get_image_path(QStringList pathlist, bool static_image) +QString AOApplication::get_image_path(QVector pathlist, bool static_image) { - QString path; - for (QString p : pathlist) { - p = get_case_sensitive_path(get_image_suffix(p, static_image)); - if (file_exists(p)) { - path = p; - break; - } + for (const VPath &p : pathlist) { + QString path = get_image_suffix(p, static_image); + if (!path.isEmpty()) { + return path; + } } - return path; + return QString(); } -QString AOApplication::get_sfx_path(QStringList pathlist) +QString AOApplication::get_sfx_path(QVector pathlist) { - QString path; - for (QString p : pathlist) { - p = get_case_sensitive_path(get_sfx_suffix(p)); - if (file_exists(p)) { - path = p; - break; + for (const VPath &p : pathlist) { + QString path = get_sfx_suffix(p); + if (!path.isEmpty()) { + return path; } } - return path; + return QString(); } + QString AOApplication::get_config_value(QString p_identifier, QString p_config, QString p_theme, QString p_subtheme, QString p_default_theme, QString p_misc) { QString path; // qDebug() << "got request for" << p_identifier << "in" << p_config; - for (QString p : get_asset_paths(p_config, p_theme, p_subtheme, p_default_theme, p_misc)) { - p = get_case_sensitive_path(p); - if (file_exists(p)) { - QSettings settings(p, QSettings::IniFormat); + for (const VPath &p : get_asset_paths(p_config, p_theme, p_subtheme, p_default_theme, p_misc)) { + path = get_real_path(p); + if (!path.isEmpty()) { + QSettings settings(path, QSettings::IniFormat); QVariant value = settings.value(p_identifier); if (value.type() == QVariant::StringList) { // qDebug() << "got" << p << "is a string list, returning" << value.toStringList().join(","); @@ -190,25 +172,22 @@ QString AOApplication::get_asset(QString p_element, QString p_theme, QString p_s return get_asset_path(get_asset_paths(p_element, p_theme, p_subtheme, p_default_theme, p_misc, p_character, p_placeholder)); } -QString AOApplication::get_image(QString p_element, QString p_theme, QString p_subtheme, QString p_default_theme, QString p_misc, QString p_character, QString p_placeholder) +QString AOApplication::get_image(QString p_element, QString p_theme, QString p_subtheme, QString p_default_theme, QString p_misc, QString p_character, QString p_placeholder, + bool static_image) { - return get_image_path(get_asset_paths(p_element, p_theme, p_subtheme, p_default_theme, p_misc, p_character, p_placeholder)); + return get_image_path(get_asset_paths(p_element, p_theme, p_subtheme, p_default_theme, p_misc, p_character, p_placeholder), static_image); } QString AOApplication::get_sfx(QString p_sfx, QString p_misc, QString p_character) { - QStringList pathlist = get_asset_paths(p_sfx, current_theme, get_subtheme(), default_theme, p_misc, p_character); + QVector pathlist = get_asset_paths(p_sfx, current_theme, get_subtheme(), default_theme, p_misc, p_character); pathlist += get_sounds_path(p_sfx); // Sounds folder path return get_sfx_path(pathlist); } QString AOApplication::get_case_sensitive_path(QString p_file) { - // no path traversal above base folder - if (!(p_file.startsWith(get_base_path()))) - return get_base_path() + p_file; - - #ifdef CASE_SENSITIVE_FILESYSTEM +#ifdef CASE_SENSITIVE_FILESYSTEM // first, check to see if it's actually there (also serves as base case for // recursion) QFileInfo file(p_file); @@ -238,3 +217,35 @@ QString AOApplication::get_case_sensitive_path(QString p_file) return p_file; #endif } + +QString AOApplication::get_real_path(const VPath &vpath) { + // Try cache first + QString phys_path = asset_lookup_cache.value(vpath); + if (!phys_path.isEmpty() && exists(phys_path)) { + return phys_path; + } + + // Cache miss; try all known mount paths + QStringList bases = get_mount_paths(); + bases.push_front(get_base_path()); + + for (const QString &base : bases) { + QDir baseDir(base); + const QString path = baseDir.absoluteFilePath(vpath.toQString()); + if (!path.startsWith(baseDir.absolutePath())) { + qWarning() << "invalid path" << path << "(path is outside vfs)"; + break; + } + if (exists(get_case_sensitive_path(path))) { + asset_lookup_cache.insert(vpath, path); + return path; + } + } + + // File or directory not found + return QString(); +} + +void AOApplication::invalidate_lookup_cache() { + asset_lookup_cache.clear(); +} -- cgit From 037d96a5d96bae6e341d8f7ca4f485d519cde02b Mon Sep 17 00:00:00 2001 From: oldmud0 Date: Sat, 5 Jun 2021 20:28:55 -0500 Subject: Use intuitive behavior for loading assets with ambiguous extensions --- src/path_functions.cpp | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) (limited to 'src/path_functions.cpp') diff --git a/src/path_functions.cpp b/src/path_functions.cpp index b1a5e48..f79f7bf 100644 --- a/src/path_functions.cpp +++ b/src/path_functions.cpp @@ -246,6 +246,39 @@ QString AOApplication::get_real_path(const VPath &vpath) { return QString(); } +// Special case of get_real_path where multiple suffixes need to be tried +// on each mount path. +QString AOApplication::get_real_suffixed_path(const VPath &vpath, + const QStringList &suffixes) { + // Try cache first + QString phys_path = asset_lookup_cache.value(vpath); + if (!phys_path.isEmpty() && exists(phys_path)) { + return phys_path; + } + + // Cache miss; try each suffix on all known mount paths + QStringList bases = get_mount_paths(); + bases.push_front(get_base_path()); + + for (const QString &base : bases) { + for (const QString &suffix : suffixes) { + QDir baseDir(base); + const QString path = baseDir.absoluteFilePath(vpath.toQString() + suffix); + if (!path.startsWith(baseDir.absolutePath())) { + qWarning() << "invalid path" << path << "(path is outside vfs)"; + break; + } + if (exists(get_case_sensitive_path(path))) { + asset_lookup_cache.insert(vpath, path); + return path; + } + } + } + + // File or directory not found + return QString(); +} + void AOApplication::invalidate_lookup_cache() { asset_lookup_cache.clear(); } -- cgit From 7a1c3f385e5fc7f0fdcde60543d32795ea64e0fc Mon Sep 17 00:00:00 2001 From: oldmud0 Date: Sat, 5 Jun 2021 22:27:56 -0500 Subject: Greatly improve case-insensitive lookup speed By using hash tables, the algorithmic complexity of the case-insensitive lookup has been reduced from O(n * k) to amortized O(k), where n is the number of files in each level and k is the number of directory levels that need to be traversed. This massively improves performance on Linux when loading characters, especially when there are many missing characters, since it is no longer necessary to scan the entire character folder repeatedly. --- src/path_functions.cpp | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) (limited to 'src/path_functions.cpp') diff --git a/src/path_functions.cpp b/src/path_functions.cpp index f79f7bf..dd65204 100644 --- a/src/path_functions.cpp +++ b/src/path_functions.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #ifdef BASE_OVERRIDE #include "base_override.h" @@ -195,6 +196,7 @@ QString AOApplication::get_case_sensitive_path(QString p_file) if (exists(p_file)) return p_file; + QString file_parent_dir = get_case_sensitive_path(file.absolutePath()); // second, does it exist in the new parent dir? @@ -204,12 +206,22 @@ QString AOApplication::get_case_sensitive_path(QString p_file) // last resort, dirlist parent dir and find case insensitive match QRegExp file_rx = QRegExp(file_basename, Qt::CaseInsensitive, QRegExp::FixedString); - QStringList files = QDir(file_parent_dir).entryList(); - int result = files.indexOf(file_rx); + static QHash listing_cache; + static QHash listing_exist_cache; - if (result != -1) - return file_parent_dir + "/" + files.at(result); + if (!listing_exist_cache.contains(qHash(file_parent_dir))) { + QStringList files = QDir(file_parent_dir).entryList(); + for (const QString &file : files) { + listing_cache.insert(qHash(file_parent_dir % QChar('/') % file.toLower()), file); + } + listing_exist_cache.insert(qHash(file_parent_dir), true); + } + QString found_file = listing_cache.value(qHash(file_parent_dir % QChar('/') % file_basename.toLower())); + + if (!found_file.isEmpty()) { + return file_parent_dir + "/" + found_file; + } // if nothing is found, let the caller handle the missing file return file_parent_dir + "/" + file_basename; -- cgit From 387233e9e3603eb7d346eb351f0c70c84a956a10 Mon Sep 17 00:00:00 2001 From: oldmud0 Date: Sun, 6 Jun 2021 22:41:40 -0500 Subject: Don't store key in asset lookup cache --- src/path_functions.cpp | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) (limited to 'src/path_functions.cpp') diff --git a/src/path_functions.cpp b/src/path_functions.cpp index dd65204..66b0a5b 100644 --- a/src/path_functions.cpp +++ b/src/path_functions.cpp @@ -204,9 +204,6 @@ QString AOApplication::get_case_sensitive_path(QString p_file) return file_parent_dir + "/" + file_basename; // last resort, dirlist parent dir and find case insensitive match - QRegExp file_rx = - QRegExp(file_basename, Qt::CaseInsensitive, QRegExp::FixedString); - static QHash listing_cache; static QHash listing_exist_cache; @@ -217,7 +214,8 @@ QString AOApplication::get_case_sensitive_path(QString p_file) } listing_exist_cache.insert(qHash(file_parent_dir), true); } - QString found_file = listing_cache.value(qHash(file_parent_dir % QChar('/') % file_basename.toLower())); + QString found_file = listing_cache.value( + qHash(file_parent_dir % QChar('/') % file_basename.toLower())); if (!found_file.isEmpty()) { return file_parent_dir + "/" + found_file; @@ -232,7 +230,7 @@ QString AOApplication::get_case_sensitive_path(QString p_file) QString AOApplication::get_real_path(const VPath &vpath) { // Try cache first - QString phys_path = asset_lookup_cache.value(vpath); + QString phys_path = asset_lookup_cache.value(qHash(vpath)); if (!phys_path.isEmpty() && exists(phys_path)) { return phys_path; } @@ -249,7 +247,7 @@ QString AOApplication::get_real_path(const VPath &vpath) { break; } if (exists(get_case_sensitive_path(path))) { - asset_lookup_cache.insert(vpath, path); + asset_lookup_cache.insert(qHash(vpath), path); return path; } } @@ -263,7 +261,7 @@ QString AOApplication::get_real_path(const VPath &vpath) { QString AOApplication::get_real_suffixed_path(const VPath &vpath, const QStringList &suffixes) { // Try cache first - QString phys_path = asset_lookup_cache.value(vpath); + QString phys_path = asset_lookup_cache.value(qHash(vpath)); if (!phys_path.isEmpty() && exists(phys_path)) { return phys_path; } @@ -281,7 +279,7 @@ QString AOApplication::get_real_suffixed_path(const VPath &vpath, break; } if (exists(get_case_sensitive_path(path))) { - asset_lookup_cache.insert(vpath, path); + asset_lookup_cache.insert(qHash(vpath), path); return path; } } -- cgit From e576a8b23d1dec48553ef4a2de3bf5c7c6925633 Mon Sep 17 00:00:00 2001 From: oldmud0 Date: Sun, 6 Jun 2021 23:27:30 -0500 Subject: Fix bug with case-sensitive vpath lookup --- src/path_functions.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'src/path_functions.cpp') diff --git a/src/path_functions.cpp b/src/path_functions.cpp index 4dc1c62..b15cc90 100644 --- a/src/path_functions.cpp +++ b/src/path_functions.cpp @@ -242,12 +242,13 @@ QString AOApplication::get_real_path(const VPath &vpath) { for (const QString &base : bases) { QDir baseDir(base); - const QString path = baseDir.absoluteFilePath(vpath.toQString()); + QString path = baseDir.absoluteFilePath(vpath.toQString()); if (!path.startsWith(baseDir.absolutePath())) { qWarning() << "invalid path" << path << "(path is outside vfs)"; break; } - if (exists(get_case_sensitive_path(path))) { + path = get_case_sensitive_path(path); + if (exists(path)) { asset_lookup_cache.insert(qHash(vpath), path); return path; } @@ -274,12 +275,13 @@ QString AOApplication::get_real_suffixed_path(const VPath &vpath, for (const QString &base : bases) { for (const QString &suffix : suffixes) { QDir baseDir(base); - const QString path = baseDir.absoluteFilePath(vpath.toQString() + suffix); + QString path = baseDir.absoluteFilePath(vpath.toQString() + suffix); if (!path.startsWith(baseDir.absolutePath())) { qWarning() << "invalid path" << path << "(path is outside vfs)"; break; } - if (exists(get_case_sensitive_path(path))) { + path = get_case_sensitive_path(path); + if (exists(path)) { asset_lookup_cache.insert(qHash(vpath), path); return path; } -- cgit From dcce1750525d0cca21ebaa7a65c1abc2ff36123c Mon Sep 17 00:00:00 2001 From: oldmud0 Date: Sun, 6 Jun 2021 23:31:48 -0500 Subject: Fix clazy-range-loop-detach warning --- src/path_functions.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/path_functions.cpp') diff --git a/src/path_functions.cpp b/src/path_functions.cpp index b15cc90..3f19715 100644 --- a/src/path_functions.cpp +++ b/src/path_functions.cpp @@ -150,7 +150,8 @@ QString AOApplication::get_config_value(QString p_identifier, QString p_config, { QString path; // qDebug() << "got request for" << p_identifier << "in" << p_config; - for (const VPath &p : get_asset_paths(p_config, p_theme, p_subtheme, p_default_theme, p_misc)) { + const auto paths = get_asset_paths(p_config, p_theme, p_subtheme, p_default_theme, p_misc); + for (const VPath &p : paths) { path = get_real_path(p); if (!path.isEmpty()) { QSettings settings(path, QSettings::IniFormat); -- cgit From 2f50cd55281cfa963ea2d6fb566b1eee675c643f Mon Sep 17 00:00:00 2001 From: oldmud0 Date: Tue, 8 Jun 2021 23:11:28 -0500 Subject: Clear case sensitive cache as well on vpath cache flush --- src/path_functions.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'src/path_functions.cpp') diff --git a/src/path_functions.cpp b/src/path_functions.cpp index 3f19715..2f5aaec 100644 --- a/src/path_functions.cpp +++ b/src/path_functions.cpp @@ -206,17 +206,15 @@ QString AOApplication::get_case_sensitive_path(QString p_file) return file_parent_dir + "/" + file_basename; // last resort, dirlist parent dir and find case insensitive match - static QHash listing_cache; - static QHash listing_exist_cache; - if (!listing_exist_cache.contains(qHash(file_parent_dir))) { + if (!dir_listing_exist_cache.contains(qHash(file_parent_dir))) { QStringList files = QDir(file_parent_dir).entryList(); for (const QString &file : files) { - listing_cache.insert(qHash(file_parent_dir % QChar('/') % file.toLower()), file); + dir_listing_cache.insert(qHash(file_parent_dir % QChar('/') % file.toLower()), file); } - listing_exist_cache.insert(qHash(file_parent_dir), true); + dir_listing_exist_cache.insert(qHash(file_parent_dir)); } - QString found_file = listing_cache.value( + QString found_file = dir_listing_cache.value( qHash(file_parent_dir % QChar('/') % file_basename.toLower())); if (!found_file.isEmpty()) { @@ -295,4 +293,6 @@ QString AOApplication::get_real_suffixed_path(const VPath &vpath, void AOApplication::invalidate_lookup_cache() { asset_lookup_cache.clear(); + dir_listing_cache.clear(); + dir_listing_exist_cache.clear(); } -- cgit From 2d3cab82c896b86b1e107b8e1773572ac0fec6d3 Mon Sep 17 00:00:00 2001 From: in1tiate Date: Thu, 22 Jul 2021 12:26:57 -0500 Subject: Fix custom objections not playing sounds --- src/path_functions.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src/path_functions.cpp') diff --git a/src/path_functions.cpp b/src/path_functions.cpp index 2f5aaec..61df1c6 100644 --- a/src/path_functions.cpp +++ b/src/path_functions.cpp @@ -264,7 +264,10 @@ QString AOApplication::get_real_suffixed_path(const VPath &vpath, // Try cache first QString phys_path = asset_lookup_cache.value(qHash(vpath)); if (!phys_path.isEmpty() && exists(phys_path)) { - return phys_path; + for (const QString &suffix : suffixes) { // make sure cached asset is the right type + if (phys_path.endsWith(suffix, Qt::CaseInsensitive)) + return phys_path; + } } // Cache miss; try each suffix on all known mount paths -- cgit From 74d01e81fbcb9e423cd18ee90eacddbb47f01b14 Mon Sep 17 00:00:00 2001 From: Salanto <62221668+Salanto@users.noreply.github.com> Date: Fri, 3 Sep 2021 01:25:28 +0200 Subject: Fix VPath lookup ignoring possibility of remote files (#588) * Fix VPath lookup ignoring possibility of remote files * Use more formal URL check Co-authored-by: oldmud0 --- src/path_functions.cpp | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'src/path_functions.cpp') diff --git a/src/path_functions.cpp b/src/path_functions.cpp index 61df1c6..6fd06e7 100644 --- a/src/path_functions.cpp +++ b/src/path_functions.cpp @@ -253,6 +253,12 @@ QString AOApplication::get_real_path(const VPath &vpath) { } } + // Not found in mount paths; check if the file is remote + QString remotePath = vpath.toQString(); + if (remotePath.startsWith("http:") || remotePath.startsWith("https:")) { + return remotePath; + } + // File or directory not found return QString(); } -- cgit