simplified handling of punctuation is spellchecker, now checks for stand alone...

simplified handling of punctuation is spellchecker, now checks for stand alone punctuation and double spaces.

src/gui/value/text.cpp

@@ -511,16 +511,9 @@ bool TextValueEditor::onChar(wxKeyEvent& ev) {
 
 // ----------------------------------------------------------------------------- : Spellchecking
 
-String spellcheck_word_at(const String& str, size_t start, String& before, String& after) {
+String spellcheck_word_at(const String& str, size_t start) {
 	size_t end = min(match_close_tag(str,start), str.size());
-	String word = untag(str.substr(start,end-start));
-	size_t start_u = 0, end_u = String::npos;
-	trim_punctuation(word, start_u, end_u);
-	before = word.substr(0,start_u);
-	after  = word.substr(end_u,String::npos);
-	word.erase(end_u,String::npos);
-	word.erase(0,start_u);
-	return word;
+	return untag(str.substr(start,end-start));
 }
 
 void spellcheck_language_at(const String& str, size_t error_pos, SpellChecker** out) {
@@ -535,8 +528,8 @@ void spellcheck_language_at(const String& str, size_t error_pos, SpellChecker**
 	out[1] = &SpellChecker::get(extra,language);
 }
 
-void get_spelling_suggestions(const String& str, size_t error_pos, vector<String>& suggestions_out, String& before, String& after) {
-	String word = spellcheck_word_at(str, error_pos, before, after);
+void get_spelling_suggestions(const String& str, size_t error_pos, vector<String>& suggestions_out) {
+	String word = spellcheck_word_at(str, error_pos);
 	// find dictionaries
 	SpellChecker* checkers[3] = {nullptr};
 	spellcheck_language_at(str, error_pos, checkers);
@@ -575,9 +568,8 @@ bool TextValueEditor::onContextMenu(IconMenu& m, wxContextMenuEvent& ev) {
 		//%m.InsertSeparator(0);
 		//%m.Insert(0,ID_SPELLING_ADD_TO_DICT, _MENU_("add to dictionary"),	_HELP_("add to dictionary"));
 		// suggestions
-		String before,after;
 		vector<String> suggestions;
-		get_spelling_suggestions(value().value(), error_pos, suggestions,before,after);
+		get_spelling_suggestions(value().value(), error_pos, suggestions);
 		// add suggestions to menu
 		m.InsertSeparator(0);
 		if (suggestions.empty()) {
@@ -585,7 +577,7 @@ bool TextValueEditor::onContextMenu(IconMenu& m, wxContextMenuEvent& ev) {
 		} else {
 			int i = 0;
 			FOR_EACH(s,suggestions) {
-				m.Insert(i, ID_SPELLING_SUGGEST + i, before+s+after, wxEmptyString);
+				m.Insert(i, ID_SPELLING_SUGGEST + i, s, wxEmptyString);
 				i++;
 			}
 		}
@@ -610,13 +602,15 @@ bool TextValueEditor::onCommand(int id) {
 	} else if (id >= ID_SPELLING_SUGGEST && id <= ID_SPELLING_SUGGEST_MAX) {
 		size_t error_pos = in_tag(value().value(), _("<error-spelling"), selection_start_i, selection_start_i);
 		if (error_pos == String::npos) throw InternalError(_("Unexpected spelling suggestion")); // wrong
-		String before,after;
+		// find the suggestions to pick from
 		vector<String> suggestions;
-		get_spelling_suggestions(value().value(), error_pos, suggestions,before,after);
+		get_spelling_suggestions(value().value(), error_pos, suggestions);
+		// select the error
 		selection_start_i = error_pos;
 		selection_end_i   = match_close_tag(value().value(), error_pos);
 		fixSelection(TYPE_INDEX);
-		replaceSelection(before + suggestions.at(id - ID_SPELLING_SUGGEST) + after, _ACTION_("correct"));
+		// replace it
+		replaceSelection(suggestions.at(id - ID_SPELLING_SUGGEST), _ACTION_("correct"));
 		return true;
 	}
 	return false;

src/script/functions/spelling.cpp

@@ -19,23 +19,12 @@ inline size_t spelled_correctly(const String& input, size_t start, size_t end, S
 	// untag
 	String word = untag(input.substr(start,end-start));
 	if (word.empty()) return true;
-	// remove punctuation
-	size_t start_u = 0, end_u = String::npos;
-	trim_punctuation(word, start_u, end_u);
-	if (start_u >= end_u) {
-		// punctuation only, but not empty => error
-		return false;
-	}
-	// find the tagged text without punctuation
-	size_t start2 = untagged_to_index(input, start_u, true, start);
-	size_t end2   = untagged_to_index(input, end_u,   true, start);
-	if (in_tag(input,_("<sym"),start2,end2) != String::npos) {
+	// symbol?
+	if (in_tag(input,_("<sym"),start,end) != String::npos) {
 		// symbols are always spelled correctly
 		return true;
 	}
 	// run through spellchecker(s)
-	word.erase(end_u,String::npos);
-	word.erase(0,start_u);
 	for (size_t i = 0 ; checkers[i] ; ++i) {
 		if (checkers[i]->spell(word)) {
 			return true;
@@ -43,7 +32,13 @@ inline size_t spelled_correctly(const String& input, size_t start, size_t end, S
 	}
 	// run through additional words regex
 	if (extra_test) {
-		ctx.setVariable(SCRIPT_VAR_input, to_script(input.substr(start2,end2-start2)));
+		// try on untagged
+		ctx.setVariable(SCRIPT_VAR_input, to_script(word));
+		if (*extra_test->eval(ctx)) {
+			return true;
+		}
+		// try on tagged
+		ctx.setVariable(SCRIPT_VAR_input, to_script(input.substr(start,end-start)));
 		if (*extra_test->eval(ctx)) {
 			return true;
 		}
@@ -59,6 +54,38 @@ void check_word(const String& tag, const String& input, String& out, size_t star
 	if (!good) out += _("</") + tag;
 }
 
+void check_word(const String& tag, const String& input, String& out, Char sep, size_t prev, size_t start, size_t end, size_t after, SpellChecker** checkers, const ScriptValueP& extra_test, Context& ctx) {
+	if (start == end) {
+		// word consisting of whitespace/punctuation only
+		if (untag(input.substr(prev,after-prev)).empty()) {
+			if (isSpace(sep) && (after == input.size() || isSpace(input.GetChar(after)))) {
+				// double space
+				out += _("<error-spelling>");
+				out.append(sep);
+				out.append(input, prev, after-end);
+				out += _("</error-spelling>");
+			} else {
+				if (sep) out.append(sep);
+				out.append(input, prev, after-prev);
+			}
+		} else {
+			// stand alone punctuation
+			if (sep) out.append(sep);
+			out += _("<error-spelling>");
+			out.append(input, prev, after-end);
+			out += _("</error-spelling>");
+		}
+	} else {
+		// before the word
+		if (sep) out.append(sep);
+		out.append(input, prev, start-prev);
+		// the word itself
+		check_word(tag, input, out, start, end, checkers, extra_test, ctx);
+		// after the word
+		out.append(input, end, after-end);
+	}
+}
+
 SCRIPT_FUNCTION(check_spelling) {
 	SCRIPT_PARAM_C(String,language);
 	SCRIPT_PARAM_C(String,input);
@@ -84,32 +111,37 @@ SCRIPT_FUNCTION(check_spelling) {
 	tag += _(">");
 	// now walk over the words in the input, and mark misspellings
 	String result;
-	size_t word_start = 0, word_end = 0, pos = 0;
+	Char sep = 0;
+	size_t prev_end = 0, word_start = 0, word_end = 0, pos = 0;
 	while (pos < input.size()) {
 		Char c = input.GetChar(pos);
 		if (c == _('<')) {
 			if (word_start == pos) {
-				// prefer to place word start inside tags
-				pos = skip_tag(input,pos);
-				result.append(input, word_start, pos - word_start);
-				word_end = word_start = pos;
+				// prefer to place word start inside tags, i.e. as late as possible
+				word_end = word_start = pos = skip_tag(input,pos);
 			} else {
 				pos = skip_tag(input,pos);
 			}
 		} else if (isSpace(c) || c == EM_DASH || c == EN_DASH) {
-			// word boundary -> check word
-			check_word(tag, input, result, word_start, word_end, checkers, extra_match, ctx);
-			// non-word characters
-			result.append(input, word_end, pos - word_end + 1);
+			// word boundary => check the word
+			check_word(tag, input, result, sep, prev_end, word_start, word_end, pos, checkers, extra_match, ctx);
 			// next
-			word_start = word_end = pos = pos + 1;
+			sep = c;
+			prev_end = word_start = word_end = pos = pos + 1;
 		} else {
-			word_end = pos = pos + 1;
+			pos++;
+			if (word_start == pos-1 && is_word_start_punctuation(c)) {
+				// skip punctuation at start of word
+				word_end = word_start = pos;
+			} else if (is_word_end_punctuation(c)) {
+				// skip punctuation at end of word
+			} else {
+				word_end = pos;
+			}
 		}
 	}
 	// last word
-	check_word(tag, input, result, word_start, word_end, checkers, extra_match, ctx);
-	result.append(input, word_end, String::npos);
+	check_word(tag, input, result, sep, prev_end, word_start, word_end, pos, checkers, extra_match, ctx);
 	// done
 	SCRIPT_RETURN(result);
 }

src/util/string.cpp

@@ -122,15 +122,22 @@ String strip_last_word(const String& s) {
 	}
 }
 
-const String word_start = String(_("[({\"\'")) + LEFT_SINGLE_QUOTE + LEFT_DOUBLE_QUOTE;
-const String word_end   = String(_("])}.,;:?!\"\'")) + RIGHT_SINGLE_QUOTE + RIGHT_DOUBLE_QUOTE;
+const String word_start_chars = String(_("[({\"\'")) + LEFT_SINGLE_QUOTE + LEFT_DOUBLE_QUOTE;
+const String word_end_chars   = String(_("])}.,;:?!\"\'")) + RIGHT_SINGLE_QUOTE + RIGHT_DOUBLE_QUOTE;
 
 void trim_punctuation(const String& str, size_t& start, size_t& end) {
-	start = str.find_first_not_of(word_start, start);
-	end   = str.find_last_not_of(word_end,    min(end,str.size()-1)) + 1;
+	start = str.find_first_not_of(word_start_chars, start);
+	end   = str.find_last_not_of(word_end_chars,    min(end,str.size()-1)) + 1;
 	if (start >= end) start = end;
 }
 
+bool is_word_start_punctuation(Char c) {
+	return word_start_chars.find_first_of(c) != String::npos;
+}
+bool is_word_end_punctuation(Char c) {
+	return word_end_chars.find_first_of(c) != String::npos;
+}
+
 // ----------------------------------------------------------------------------- : Caseing
 
 /// Quick check to see if the substring starting at the given iterator is equal to some given string

src/util/string.hpp

@@ -139,6 +139,9 @@ String strip_last_word(const String&);
 /// Trim punctuation at the start/end of a word in the range [start..end)
 void trim_punctuation(const String&, size_t& start, size_t& end);
 
+bool is_word_start_punctuation(Char c);
+bool is_word_end_punctuation(Char c);
+
 // ----------------------------------------------------------------------------- : Caseing
 
 /// Make each word in a string start with an upper case character.