From 994402ec9ae6216f0f8193887ca351225dcc7e42 Mon Sep 17 00:00:00 2001 From: tangxifan Date: Mon, 2 Jan 2023 12:38:16 -0800 Subject: [PATCH 1/2] [engine] move shell cmd split function to openfpga tokenizer --- libs/libopenfpgashell/src/shell.tpp | 24 ++----------- .../src/openfpga_tokenizer.cpp | 34 +++++++++++++++++++ libs/libopenfpgautil/src/openfpga_tokenizer.h | 22 ++++++++++++ 3 files changed, 59 insertions(+), 21 deletions(-) diff --git a/libs/libopenfpgashell/src/shell.tpp b/libs/libopenfpgashell/src/shell.tpp index a69e8edef..0426b5949 100644 --- a/libs/libopenfpgashell/src/shell.tpp +++ b/libs/libopenfpgashell/src/shell.tpp @@ -487,13 +487,9 @@ template int Shell::execute_command(const char* cmd_line, T& common_context) { openfpga::StringToken tokenizer(cmd_line); + tokenizer.add_delim(' '); /* Do not split the string in each quote "", as they should be a piece */ - std::vector quote_anchors; - size_t quote_found = tokenizer.data().find("\""); - while (std::string::npos != quote_found) { - quote_anchors.push_back(quote_found); - quote_found = tokenizer.data().find("\"", quote_found+1); - } + std::vector quote_anchors = tokenizer.find_positions('\"'); /* Quote should be not be started with! */ if (!quote_anchors.empty() && quote_anchors.front() == 0) { VTR_LOG("Quotes (\") should NOT be the first charactor in command line: '%s'\n", cmd_line); @@ -509,21 +505,7 @@ int Shell::execute_command(const char* cmd_line, if (quote_anchors.empty()) { tokens = tokenizer.split(" "); } else { - /* There are pairs of quotes, identify the chunk which should be split*/ - std::vector token_chunks = tokenizer.split("\""); - for (size_t ichunk = 0; ichunk < token_chunks.size(); ichunk++) { - /* Chunk with even index (including the first) is always out of two quote -> Split! - * Chunk with odd index is always between two quotes -> Do not split! - */ - if (ichunk % 2 == 0) { - openfpga::StringToken chunk_tokenizer(token_chunks[ichunk]); - for (std::string curr_token : chunk_tokenizer.split(" ")) { - tokens.push_back(curr_token); - } - } else { - tokens.push_back(token_chunks[ichunk]); - } - } + tokens = tokenizer.split_by_chunks('\"'); } /* Find if the command name is valid */ diff --git a/libs/libopenfpgautil/src/openfpga_tokenizer.cpp b/libs/libopenfpgautil/src/openfpga_tokenizer.cpp index 21e4449a5..a4382ac09 100644 --- a/libs/libopenfpgautil/src/openfpga_tokenizer.cpp +++ b/libs/libopenfpgautil/src/openfpga_tokenizer.cpp @@ -89,6 +89,40 @@ std::vector StringToken::split() { return split(delims); } +std::vector StringToken::find_positions(const char& delim) const { + std::vector anchors; + size_t found = data_.find(delim); + while (std::string::npos != found) { + anchors.push_back(found); + found = data_.find(delim, found + 1); + } + return anchors; +} + +std::vector StringToken::split_by_chunks(const char& chunk_delim, const bool& split_odd_chunk) const { + size_t chunk_idx_mod = 0; + if (split_odd_chunk) { + chunk_idx_mod = 1; + } + std::vector tokens; + /* There are pairs of quotes, identify the chunk which should be split*/ + std::vector token_chunks = split(chunk_delim); + for (size_t ichunk = 0; ichunk < token_chunks.size(); ichunk++) { + /* Chunk with even index (including the first) is always out of two quote -> Split! + * Chunk with odd index is always between two quotes -> Do not split! + */ + if (ichunk % 2 == chunk_idx_mod) { + StringToken chunk_tokenizer(token_chunks[ichunk]); + for (std::string curr_token : chunk_tokenizer.split()) { + tokens.push_back(curr_token); + } + } else { + tokens.push_back(token_chunks[ichunk]); + } + } + return tokens; +} + /************************************************************************ * Public Mutators ***********************************************************************/ diff --git a/libs/libopenfpgautil/src/openfpga_tokenizer.h b/libs/libopenfpgautil/src/openfpga_tokenizer.h index af803c7d6..dd71ed872 100644 --- a/libs/libopenfpgautil/src/openfpga_tokenizer.h +++ b/libs/libopenfpgautil/src/openfpga_tokenizer.h @@ -27,6 +27,28 @@ class StringToken { std::vector split(const char* delim) const; std::vector split(const std::vector& delim) const; std::vector split(); + /** @brief Find the position (i-th charactor) in a string for a given delimiter, it will return a list of positions + * For example, to find the position of all quotes (") in a string: + * "we" are good + * The following code is suggested: + * StringToken tokenizer("\"we\" are good"); + * std::vector anchors = tokenizer.find_positions('\"') + * The following vector will be returned: + * [0, 3] */ + std::vector find_positions(const char& delim) const; + /** @brief split the string for each chunk. This is useful where there are chunks of substring should not be splitted by the given delimiter + * For example, to split the string with quotes (") in a string: + * source "cmdA --opt1 val1;cmdB --opt2 val2" --verbose + * where the string between the two quotes should not be splitted + * The following code is suggested: + * StringToken tokenizer("source \"cmdA --opt1 val1;cmdB --opt2 val2\" --verbose"); + * std::vector tokenizer.split_by_chunks('\"', true); + * The following vector will be returned: + * ["source" "cmdA --opt1 val1;cmdB --opt2 val2" "--verbose"] + * + * .. note:: The option ``split_odd_chunk`` is useful when the chunk delimiter appears at the beginning of the string. + */ + std::vector split_by_chunks(const char& chunk_delim, const bool& split_odd_chunk = false) const; public: /* Public Mutators */ void set_data(const std::string& data); From 77b64a21d4d4d644180c61a981bb1ce1468fd92e Mon Sep 17 00:00:00 2001 From: tangxifan Date: Mon, 2 Jan 2023 12:41:24 -0800 Subject: [PATCH 2/2] [lib] format --- .../src/openfpga_tokenizer.cpp | 11 +++--- libs/libopenfpgautil/src/openfpga_tokenizer.h | 36 +++++++++---------- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/libs/libopenfpgautil/src/openfpga_tokenizer.cpp b/libs/libopenfpgautil/src/openfpga_tokenizer.cpp index a4382ac09..678f853b9 100644 --- a/libs/libopenfpgautil/src/openfpga_tokenizer.cpp +++ b/libs/libopenfpgautil/src/openfpga_tokenizer.cpp @@ -99,20 +99,21 @@ std::vector StringToken::find_positions(const char& delim) const { return anchors; } -std::vector StringToken::split_by_chunks(const char& chunk_delim, const bool& split_odd_chunk) const { +std::vector StringToken::split_by_chunks( + const char& chunk_delim, const bool& split_odd_chunk) const { size_t chunk_idx_mod = 0; if (split_odd_chunk) { chunk_idx_mod = 1; } - std::vector tokens; + std::vector tokens; /* There are pairs of quotes, identify the chunk which should be split*/ std::vector token_chunks = split(chunk_delim); for (size_t ichunk = 0; ichunk < token_chunks.size(); ichunk++) { - /* Chunk with even index (including the first) is always out of two quote -> Split! - * Chunk with odd index is always between two quotes -> Do not split! + /* Chunk with even index (including the first) is always out of two quote -> + * Split! Chunk with odd index is always between two quotes -> Do not split! */ if (ichunk % 2 == chunk_idx_mod) { - StringToken chunk_tokenizer(token_chunks[ichunk]); + StringToken chunk_tokenizer(token_chunks[ichunk]); for (std::string curr_token : chunk_tokenizer.split()) { tokens.push_back(curr_token); } diff --git a/libs/libopenfpgautil/src/openfpga_tokenizer.h b/libs/libopenfpgautil/src/openfpga_tokenizer.h index dd71ed872..48470bbe4 100644 --- a/libs/libopenfpgautil/src/openfpga_tokenizer.h +++ b/libs/libopenfpgautil/src/openfpga_tokenizer.h @@ -27,28 +27,28 @@ class StringToken { std::vector split(const char* delim) const; std::vector split(const std::vector& delim) const; std::vector split(); - /** @brief Find the position (i-th charactor) in a string for a given delimiter, it will return a list of positions - * For example, to find the position of all quotes (") in a string: - * "we" are good - * The following code is suggested: - * StringToken tokenizer("\"we\" are good"); - * std::vector anchors = tokenizer.find_positions('\"') - * The following vector will be returned: - * [0, 3] */ + /** @brief Find the position (i-th charactor) in a string for a given + * delimiter, it will return a list of positions For example, to find the + * position of all quotes (") in a string: "we" are good The following code is + * suggested: StringToken tokenizer("\"we\" are good"); std::vector + * anchors = tokenizer.find_positions('\"') The following vector will be + * returned: [0, 3] */ std::vector find_positions(const char& delim) const; - /** @brief split the string for each chunk. This is useful where there are chunks of substring should not be splitted by the given delimiter - * For example, to split the string with quotes (") in a string: - * source "cmdA --opt1 val1;cmdB --opt2 val2" --verbose - * where the string between the two quotes should not be splitted - * The following code is suggested: - * StringToken tokenizer("source \"cmdA --opt1 val1;cmdB --opt2 val2\" --verbose"); + /** @brief split the string for each chunk. This is useful where there are + * chunks of substring should not be splitted by the given delimiter For + * example, to split the string with quotes (") in a string: source "cmdA + * --opt1 val1;cmdB --opt2 val2" --verbose where the string between the two + * quotes should not be splitted The following code is suggested: StringToken + * tokenizer("source \"cmdA --opt1 val1;cmdB --opt2 val2\" --verbose"); * std::vector tokenizer.split_by_chunks('\"', true); - * The following vector will be returned: + * The following vector will be returned: * ["source" "cmdA --opt1 val1;cmdB --opt2 val2" "--verbose"] - * - * .. note:: The option ``split_odd_chunk`` is useful when the chunk delimiter appears at the beginning of the string. + * + * .. note:: The option ``split_odd_chunk`` is useful when the chunk delimiter + * appears at the beginning of the string. */ - std::vector split_by_chunks(const char& chunk_delim, const bool& split_odd_chunk = false) const; + std::vector split_by_chunks( + const char& chunk_delim, const bool& split_odd_chunk = false) const; public: /* Public Mutators */ void set_data(const std::string& data);