From e947a29fc88f098febd2d93a8d9acf8ccedd4229 Mon Sep 17 00:00:00 2001 From: Vojtech Bubnik Date: Thu, 12 Aug 2021 15:27:32 +0200 Subject: [PATCH] Follow-up to 7c01ddf996f4b8ca6d7e71c001e7650b3bf14609 1) Starting with this commit, configuration block exported into G-code is delimited by "; prusaslicer_config = begin" and "; prusaslicer_config = end". These delimiters look like any other key / value configuration pairs on purpose to be compatible with older PrusaSlicer config parsing from G-code. 2) Config parser from G-code newly searches for "; generated by ..." comment over the complete G-code, thus it is compatible with various post processing scripts extending the G-code at the start. 3) Config parser from G-code parses PrusaSlicer version from the "; generated by PrusaSlicer ...." header and if the G-code was generated by PrusaSlicer 2.4.0-alpha0 and newer, it expects that the G-code already contains the "; prusaslicer_config = begin / end" tags and it relies on these tags to extract configuration. 4) A new simple and robust parser was written for reading project configuration from 3MF / AMF, while a heuristic parser to read config from G-code located at the end of the G-code file was used before. --- src/libslic3r/Config.cpp | 255 +++++++++++++++++++++---- src/libslic3r/Config.hpp | 8 +- src/libslic3r/Format/3mf.cpp | 4 +- src/libslic3r/Format/AMF.cpp | 4 +- src/libslic3r/GCode.cpp | 6 +- src/libslic3r/GCode/GCodeProcessor.cpp | 2 +- src/libslic3r/PresetBundle.cpp | 2 +- src/libslic3r/Semver.hpp | 15 +- 8 files changed, 251 insertions(+), 45 deletions(-) diff --git a/src/libslic3r/Config.cpp b/src/libslic3r/Config.cpp index 8fb774cb9..41ee9231f 100644 --- a/src/libslic3r/Config.cpp +++ b/src/libslic3r/Config.cpp @@ -625,7 +625,7 @@ void ConfigBase::setenv_() const ConfigSubstitutions ConfigBase::load(const std::string &file, ForwardCompatibilitySubstitutionRule compatibility_rule) { return is_gcode_file(file) ? - this->load_from_gcode_file(file, true /* check header */, compatibility_rule) : + this->load_from_gcode_file(file, compatibility_rule) : this->load_from_ini(file, compatibility_rule); } @@ -637,6 +637,54 @@ ConfigSubstitutions ConfigBase::load_from_ini(const std::string &file, ForwardCo return this->load(tree, compatibility_rule); } +ConfigSubstitutions ConfigBase::load_from_ini_string(const std::string &data, ForwardCompatibilitySubstitutionRule compatibility_rule) +{ + boost::property_tree::ptree tree; + std::istringstream iss(data); + boost::property_tree::read_ini(iss, tree); + return this->load(tree, compatibility_rule); +} + +// Loading a "will be one day a legacy format" of configuration stored into 3MF or AMF. +// Accepts the same data as load_from_ini_string(), only with each configuration line possibly prefixed with a semicolon (G-code comment). +ConfigSubstitutions ConfigBase::load_from_ini_string_commented(std::string &&data, ForwardCompatibilitySubstitutionRule compatibility_rule) +{ + // Convert the "data" string into INI format by removing the semi-colons at the start of a line. + // Also the "; generated by PrusaSlicer ..." comment line will be removed. + size_t j = 0; + for (size_t i = 0; i < data.size();) + if (i == 0 || data[i] == '\n') { + // Start of a line. + if (i != 0) { + // Consume LF. + assert(data[i] == '\n'); + // Don't keep empty lines. + if (j != 0 && data[j] != '\n') + data[j ++] = data[i ++]; + } + // Skip all leading spaces; + for (; i < data.size() && (data[i] == ' ' || data[i] == '\t'); ++ i) ; + // Skip the semicolon (comment indicator). + if (i < data.size() && data[i] == ';') + ++ i; + // Skip all leading spaces after semicolon. + for (; i < data.size() && (data[i] == ' ' || data[i] == '\t'); ++ i) ; + if (strncmp(data.data() + i, "generated by ", 13) == 0) { + // Skip the "; generated by ..." line. + for (; i < data.size() && data[i] != '\n'; ++ i); + } + } else if (data[i] == '\r' && i + 1 < data.size() && data[i + 1] == '\n') { + // Skip CR. + ++ i; + } else { + // Consume the rest of the data. + data[j ++] = data[i ++]; + } + data.erase(data.begin() + j, data.end()); + + return this->load_from_ini_string(data, compatibility_rule); +} + ConfigSubstitutions ConfigBase::load(const boost::property_tree::ptree &tree, ForwardCompatibilitySubstitutionRule compatibility_rule) { ConfigSubstitutionContext substitutions_ctxt(compatibility_rule); @@ -651,37 +699,8 @@ ConfigSubstitutions ConfigBase::load(const boost::property_tree::ptree &tree, Fo return std::move(substitutions_ctxt.substitutions); } -// Load the config keys from the tail of a G-code file. -ConfigSubstitutions ConfigBase::load_from_gcode_file(const std::string &file, bool check_header, ForwardCompatibilitySubstitutionRule compatibility_rule) -{ - // Read a 64k block from the end of the G-code. - boost::nowide::ifstream ifs(file); - if (check_header) { - const char slic3r_gcode_header[] = "; generated by Slic3r "; - const char prusaslicer_gcode_header[] = "; generated by PrusaSlicer "; - std::string firstline; - std::getline(ifs, firstline); - if (strncmp(slic3r_gcode_header, firstline.c_str(), strlen(slic3r_gcode_header)) != 0 && - strncmp(prusaslicer_gcode_header, firstline.c_str(), strlen(prusaslicer_gcode_header)) != 0) - throw Slic3r::RuntimeError("Not a PrusaSlicer / Slic3r PE generated g-code."); - } - ifs.seekg(0, ifs.end); - auto file_length = ifs.tellg(); - auto data_length = std::min(65535, file_length); - ifs.seekg(file_length - data_length, ifs.beg); - std::vector data(size_t(data_length) + 1, 0); - ifs.read(data.data(), data_length); - ifs.close(); - - ConfigSubstitutionContext substitutions_ctxt(compatibility_rule); - size_t key_value_pairs = load_from_gcode_string(data.data(), substitutions_ctxt); - if (key_value_pairs < 80) - throw Slic3r::RuntimeError(format("Suspiciously low number of configuration values extracted from %1%: %2%", file, key_value_pairs)); - return std::move(substitutions_ctxt.substitutions); -} - // Load the config keys from the given string. -size_t ConfigBase::load_from_gcode_string(const char* str, ConfigSubstitutionContext& substitutions) +static inline size_t load_from_gcode_string_legacy(ConfigBase &config, const char *str, ConfigSubstitutionContext &substitutions) { if (str == nullptr) return 0; @@ -704,7 +723,7 @@ size_t ConfigBase::load_from_gcode_string(const char* str, ConfigSubstitutionCon if (end - (++ start) < 10 || start[0] != ';' || start[1] != ' ') break; const char *key = start + 2; - if (!(*key >= 'a' && *key <= 'z') || (*key >= 'A' && *key <= 'Z')) + if (!((*key >= 'a' && *key <= 'z') || (*key >= 'A' && *key <= 'Z'))) // A key must start with a letter. break; const char *sep = key; @@ -726,7 +745,7 @@ size_t ConfigBase::load_from_gcode_string(const char* str, ConfigSubstitutionCon if (key == nullptr) break; try { - this->set_deserialize(std::string(key, key_end), std::string(value, end), substitutions); + config.set_deserialize(std::string(key, key_end), std::string(value, end), substitutions); ++num_key_value_pairs; } catch (UnknownOptionException & /* e */) { @@ -735,7 +754,175 @@ size_t ConfigBase::load_from_gcode_string(const char* str, ConfigSubstitutionCon end = start; } - return num_key_value_pairs; + return num_key_value_pairs; +} + +// Reading a config from G-code back to front for performance reasons: We don't want to scan +// hundreds of MB file for a short config block, which we expect to find at the end of the G-code. +class ReverseLineReader +{ +public: + using pos_type = boost::nowide::ifstream::pos_type; + + // Stop at file_start + ReverseLineReader(boost::nowide::ifstream &ifs, pos_type file_start) : m_ifs(ifs), m_file_start(file_start) + { + m_ifs.seekg(0, m_ifs.end); + m_file_pos = m_ifs.tellg(); + m_block.assign(m_block_size, 0); + } + + bool getline(std::string &out) { + out.clear(); + for (;;) { + if (m_block_len == 0) { + // Read the next block. + m_block_len = size_t(std::min(m_block_size, m_file_pos - m_file_start)); + if (m_block_len == 0) + return false; + m_file_pos -= m_block_len; + m_ifs.seekg(m_file_pos, m_ifs.beg); + if (! m_ifs.read(m_block.data(), m_block_len)) + return false; + } + + assert(m_block_len > 0); + // Non-empty buffer. Find another LF. + int i = int(m_block_len) - 1; + for (; i >= 0; -- i) + if (m_block[i] == '\n') + break; + // i is position of LF or -1 if not found. + if (i == -1) { + // LF not found. Just make a backup of the buffer and continue. + out.insert(out.begin(), m_block.begin(), m_block.begin() + m_block_len); + m_block_len = 0; + } else { + assert(i >= 0); + // Copy new line to the output. It may be empty. + out.insert(out.begin(), m_block.begin() + i + 1, m_block.begin() + m_block_len); + // Block length without the newline. + m_block_len = i; + // Remove CRLF from the end of the block. + if (m_block_len > 0 && m_block[m_block_len - 1] == '\r') + -- m_block_len; + return true; + } + } + assert(false); + return false; + } + +private: + boost::nowide::ifstream &m_ifs; + std::vector m_block; + size_t m_block_size = 65536; + size_t m_block_len = 0; + pos_type m_file_start; + pos_type m_file_pos = 0; +}; + +// Load the config keys from the tail of a G-code file. +ConfigSubstitutions ConfigBase::load_from_gcode_file(const std::string &file, ForwardCompatibilitySubstitutionRule compatibility_rule) +{ + // Read a 64k block from the end of the G-code. + boost::nowide::ifstream ifs(file); + // Look for Slic3r or PrusaSlicer header. + // Look for the header across the whole file as the G-code may have been extended at the start by a post-processing script or the user. + bool has_delimiters = false; + { + static constexpr const char slic3r_gcode_header[] = "; generated by Slic3r "; + static constexpr const char prusaslicer_gcode_header[] = "; generated by PrusaSlicer "; + std::string header; + bool header_found = false; + while (std::getline(ifs, header)) { + if (strncmp(slic3r_gcode_header, header.c_str(), strlen(slic3r_gcode_header)) == 0) { + header_found = true; + break; + } else if (strncmp(prusaslicer_gcode_header, header.c_str(), strlen(prusaslicer_gcode_header)) == 0) { + // Parse PrusaSlicer version. + size_t i = strlen(prusaslicer_gcode_header); + for (; i < header.size() && header[i] == ' '; ++ i) ; + size_t j = i; + for (; j < header.size() && header[j] != ' '; ++ j) ; + try { + Semver semver(header.substr(i, j - i)); + has_delimiters = semver >= Semver(2, 4, 0, nullptr, "alpha0"); + } catch (const RuntimeError &) { + } + header_found = true; + break; + } + } + if (! header_found) + throw Slic3r::RuntimeError("Not a PrusaSlicer / Slic3r PE generated g-code."); + } + + auto header_end_pos = ifs.tellg(); + ConfigSubstitutionContext substitutions_ctxt(compatibility_rule); + size_t key_value_pairs = 0; + + if (has_delimiters) + { + // PrusaSlicer starting with 2.4.0-alpha0 delimits the config section stored into G-code with + // ; prusaslicer_config = begin + // ... + // ; prusaslicer_config = end + // The begin / end tags look like any other key / value pairs on purpose to be compatible with older G-code viewer. + // Read the file in reverse line by line. + ReverseLineReader reader(ifs, header_end_pos); + // Read the G-code file by 64k blocks back to front. + bool begin_found = false; + bool end_found = false; + std::string line; + while (reader.getline(line)) + if (line == "; prusaslicer_config = end") { + end_found = true; + break; + } + if (! end_found) + throw Slic3r::RuntimeError(format("Configuration block closing tag \"; prusaslicer_config = end\" not found when reading %1%", file)); + std::string key, value; + while (reader.getline(line)) { + if (line == "; prusaslicer_config = begin") { + begin_found = true; + break; + } + // line should be a valid key = value pair. + auto pos = line.find('='); + if (pos != std::string::npos && pos > 1 && line.front() == ';') { + key = line.substr(1, pos - 1); + value = line.substr(pos + 1); + boost::trim(key); + boost::trim(value); + try { + this->set_deserialize(key, value, substitutions_ctxt); + ++ key_value_pairs; + } catch (UnknownOptionException & /* e */) { + // ignore + } + } + } + if (! begin_found) + throw Slic3r::RuntimeError(format("Configuration block opening tag \"; prusaslicer_config = begin\" not found when reading %1%", file)); + } + else + { + // Slic3r or PrusaSlicer older than 2.4.0-alpha0 do not emit any delimiter. + // Try a heuristics reading the G-code from back. + ifs.seekg(0, ifs.end); + auto file_length = ifs.tellg(); + auto data_length = std::min(65535, file_length - header_end_pos); + ifs.seekg(file_length - data_length, ifs.beg); + std::vector data(size_t(data_length) + 1, 0); + ifs.read(data.data(), data_length); + ifs.close(); + key_value_pairs = load_from_gcode_string_legacy(*this, data.data(), substitutions_ctxt); + } + + if (key_value_pairs < 80) + throw Slic3r::RuntimeError(format("Suspiciously low number of configuration values extracted from %1%: %2%", file, key_value_pairs)); + return std::move(substitutions_ctxt.substitutions); } void ConfigBase::save(const std::string &file) const diff --git a/src/libslic3r/Config.hpp b/src/libslic3r/Config.hpp index 04355bd27..8690f2eb5 100644 --- a/src/libslic3r/Config.hpp +++ b/src/libslic3r/Config.hpp @@ -1934,9 +1934,11 @@ public: void setenv_() const; ConfigSubstitutions load(const std::string &file, ForwardCompatibilitySubstitutionRule compatibility_rule); ConfigSubstitutions load_from_ini(const std::string &file, ForwardCompatibilitySubstitutionRule compatibility_rule); - ConfigSubstitutions load_from_gcode_file(const std::string &file, bool check_header /* = true */, ForwardCompatibilitySubstitutionRule compatibility_rule); - // Returns number of key/value pairs extracted. - size_t load_from_gcode_string(const char* str, ConfigSubstitutionContext& substitutions); + ConfigSubstitutions load_from_ini_string(const std::string &data, ForwardCompatibilitySubstitutionRule compatibility_rule); + // Loading a "will be one day a legacy format" of configuration stored into 3MF or AMF. + // Accepts the same data as load_from_ini_string(), only with each configuration line possibly prefixed with a semicolon (G-code comment). + ConfigSubstitutions load_from_ini_string_commented(std::string &&data, ForwardCompatibilitySubstitutionRule compatibility_rule); + ConfigSubstitutions load_from_gcode_file(const std::string &file, ForwardCompatibilitySubstitutionRule compatibility_rule); ConfigSubstitutions load(const boost::property_tree::ptree &tree, ForwardCompatibilitySubstitutionRule compatibility_rule); void save(const std::string &file) const; diff --git a/src/libslic3r/Format/3mf.cpp b/src/libslic3r/Format/3mf.cpp index 16d86ac28..2a76f218f 100644 --- a/src/libslic3r/Format/3mf.cpp +++ b/src/libslic3r/Format/3mf.cpp @@ -875,7 +875,9 @@ namespace Slic3r { add_error("Error while reading config data to buffer"); return; } - config.load_from_gcode_string(buffer.data(), config_substitutions); + //FIXME Loading a "will be one day a legacy format" of configuration in a form of a G-code comment. + // Each config line is prefixed with a semicolon (G-code comment), that is ugly. + config_substitutions.substitutions = config.load_from_ini_string_commented(std::move(buffer), config_substitutions.rule); } } diff --git a/src/libslic3r/Format/AMF.cpp b/src/libslic3r/Format/AMF.cpp index 0312c7f22..35b3e0cf4 100644 --- a/src/libslic3r/Format/AMF.cpp +++ b/src/libslic3r/Format/AMF.cpp @@ -706,7 +706,9 @@ void AMFParserContext::endElement(const char * /* name */) case NODE_TYPE_METADATA: if ((m_config != nullptr) && strncmp(m_value[0].c_str(), SLIC3R_CONFIG_TYPE, strlen(SLIC3R_CONFIG_TYPE)) == 0) { - m_config->load_from_gcode_string(m_value[1].c_str(), *m_config_substitutions); + //FIXME Loading a "will be one day a legacy format" of configuration in a form of a G-code comment. + // Each config line is prefixed with a semicolon (G-code comment), that is ugly. + m_config_substitutions->substitutions = m_config->load_from_ini_string_commented(std::move(m_value[1].c_str()), m_config_substitutions->rule); } else if (strncmp(m_value[0].c_str(), "slic3r.", 7) == 0) { const char *opt_key = m_value[0].c_str() + 7; diff --git a/src/libslic3r/GCode.cpp b/src/libslic3r/GCode.cpp index abdbafd31..254f1d4fd 100644 --- a/src/libslic3r/GCode.cpp +++ b/src/libslic3r/GCode.cpp @@ -1464,13 +1464,15 @@ void GCode::_do_export(Print& print, FILE* file, ThumbnailsGeneratorCallback thu _write_format(file, "; total toolchanges = %i\n", print.m_print_statistics.total_toolchanges); _write_format(file, ";%s\n", GCodeProcessor::reserved_tag(GCodeProcessor::ETags::Estimated_Printing_Time_Placeholder).c_str()); - // Append full config. - _write(file, "\n"); + // Append full config, delimited by two 'phony' configuration keys prusaslicer_config = begin and prusaslicer_config = end. + // The delimiters are structured as configuration key / value pairs to be parsable by older versions of PrusaSlicer G-code viewer. { + _write(file, "\n; prusaslicer_config = begin\n"); std::string full_config; append_full_config(print, full_config); if (!full_config.empty()) _write(file, full_config); + _write(file, "; prusaslicer_config = end\n"); } print.throw_if_canceled(); } diff --git a/src/libslic3r/GCode/GCodeProcessor.cpp b/src/libslic3r/GCode/GCodeProcessor.cpp index 1141ca2a7..b5c10823e 100644 --- a/src/libslic3r/GCode/GCodeProcessor.cpp +++ b/src/libslic3r/GCode/GCodeProcessor.cpp @@ -1172,7 +1172,7 @@ void GCodeProcessor::process_file(const std::string& filename, bool apply_postpr // Silently substitute unknown values by new ones for loading configurations from PrusaSlicer's own G-code. // Showing substitution log or errors may make sense, but we are not really reading many values from the G-code config, // thus a probability of incorrect substitution is low and the G-code viewer is a consumer-only anyways. - config.load_from_gcode_file(filename, false, ForwardCompatibilitySubstitutionRule::EnableSilent); + config.load_from_gcode_file(filename, ForwardCompatibilitySubstitutionRule::EnableSilent); apply_config(config); } else if (m_producer == EProducer::Simplify3D) diff --git a/src/libslic3r/PresetBundle.cpp b/src/libslic3r/PresetBundle.cpp index b2e35fa2e..9f089ea1d 100644 --- a/src/libslic3r/PresetBundle.cpp +++ b/src/libslic3r/PresetBundle.cpp @@ -700,7 +700,7 @@ ConfigSubstitutions PresetBundle::load_config_file(const std::string &path, Forw if (is_gcode_file(path)) { DynamicPrintConfig config; config.apply(FullPrintConfig::defaults()); - ConfigSubstitutions config_substitutions = config.load_from_gcode_file(path, true /* check_header */, compatibility_rule); + ConfigSubstitutions config_substitutions = config.load_from_gcode_file(path, compatibility_rule); Preset::normalize(config); load_config_file_config(path, true, std::move(config)); return config_substitutions; diff --git a/src/libslic3r/Semver.hpp b/src/libslic3r/Semver.hpp index f55fa9f9f..45d2bac1c 100644 --- a/src/libslic3r/Semver.hpp +++ b/src/libslic3r/Semver.hpp @@ -25,8 +25,17 @@ public: Semver() : ver(semver_zero()) {} Semver(int major, int minor, int patch, - boost::optional metadata = boost::none, - boost::optional prerelease = boost::none) + boost::optional metadata, boost::optional prerelease) + : ver(semver_zero()) + { + ver.major = major; + ver.minor = minor; + ver.patch = patch; + set_metadata(metadata); + set_prerelease(prerelease); + } + + Semver(int major, int minor, int patch, const char *metadata = nullptr, const char *prerelease = nullptr) : ver(semver_zero()) { ver.major = major; @@ -102,7 +111,9 @@ public: void set_min(int min) { ver.minor = min; } void set_patch(int patch) { ver.patch = patch; } void set_metadata(boost::optional meta) { ver.metadata = meta ? strdup(*meta) : nullptr; } + void set_metadata(const char *meta) { ver.metadata = meta ? strdup(meta) : nullptr; } void set_prerelease(boost::optional pre) { ver.prerelease = pre ? strdup(*pre) : nullptr; } + void set_prerelease(const char *pre) { ver.prerelease = pre ? strdup(pre) : nullptr; } // Comparison bool operator<(const Semver &b) const { return ::semver_compare(ver, b.ver) == -1; }