Fixed: [Issue 91] UTF-16 characters in strings results in syntax error.
diff --git a/com.google.eclipse.protobuf.test/src/com/google/eclipse/protobuf/bugs/Issue91AddSupportForUTF16Strings.java b/com.google.eclipse.protobuf.test/src/com/google/eclipse/protobuf/bugs/Issue91AddSupportForUTF16Strings.java
new file mode 100644
index 0000000..e505ffd
--- /dev/null
+++ b/com.google.eclipse.protobuf.test/src/com/google/eclipse/protobuf/bugs/Issue91AddSupportForUTF16Strings.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2011 Google Inc.
+ *
+ * All rights reserved. This program and the accompanying materials are made available under the terms of the Eclipse
+ * Public License v1.0 which accompanies this distribution, and is available at
+ *
+ * http://www.eclipse.org/legal/epl-v10.html
+ */
+package com.google.eclipse.protobuf.bugs;
+
+import com.google.eclipse.protobuf.junit.core.XtextRule;
+import com.google.eclipse.protobuf.junit.util.MultiLineTextBuilder;
+
+import org.junit.*;
+
+/**
+ * Tests fix for <a href="http://code.google.com/p/protobuf-dt/issues/detail?id=91">Issue 91</a>.
+ *
+ * @author alruiz@google.com (Alex Ruiz)
+ */
+public class Issue91AddSupportForUTF16Strings {
+
+ @Rule public XtextRule xtext = new XtextRule();
+
+ @Test public void should_recognize_UTF16_strings() {
+ MultiLineTextBuilder proto = new MultiLineTextBuilder();
+ proto.append("message Foo { ")
+ .append(" optional string bar = 1 [default=\"\\302\\265\"];")
+ .append("} ");
+ xtext.parse(proto);
+ }
+}
diff --git a/com.google.eclipse.protobuf/src/com/google/eclipse/protobuf/Protobuf.xtext b/com.google.eclipse.protobuf/src/com/google/eclipse/protobuf/Protobuf.xtext
index df250b8..d524c02 100644
--- a/com.google.eclipse.protobuf/src/com/google/eclipse/protobuf/Protobuf.xtext
+++ b/com.google.eclipse.protobuf/src/com/google/eclipse/protobuf/Protobuf.xtext
@@ -152,8 +152,11 @@
SL_STRING (SL_STRING)*;
terminal SL_STRING:
- '"' ('\\' ('b' | 't' | 'n' | 'f' | 'r' | 'u' | '"' | "'" | '\\') | !('\\' | '"'))* '"' (WS)*;
+ '"' ('\\' ('b' | 't' | 'n' | 'f' | 'r' | 'u' | '"' | "'" | '\\' | UNICODE_OCTAL) | !('\\' | '"'))* '"' (WS)*;
+terminal UNICODE_OCTAL:
+ ('0'..'3')('0'..'7')('0'..'7');
+
Nan:
number='nan';
diff --git a/com.google.eclipse.protobuf/src/com/google/eclipse/protobuf/conversion/STRINGValueConverter.java b/com.google.eclipse.protobuf/src/com/google/eclipse/protobuf/conversion/STRINGValueConverter.java
index f8030dc..63c5f96 100644
--- a/com.google.eclipse.protobuf/src/com/google/eclipse/protobuf/conversion/STRINGValueConverter.java
+++ b/com.google.eclipse.protobuf/src/com/google/eclipse/protobuf/conversion/STRINGValueConverter.java
@@ -14,6 +14,8 @@
import org.eclipse.xtext.conversion.impl.AbstractLexerBasedConverter;
import org.eclipse.xtext.nodemodel.INode;
+import java.util.regex.Pattern;
+
/**
* Converts multi-line strings to {@code String}s.
*
@@ -21,6 +23,8 @@
*/
public class STRINGValueConverter extends AbstractLexerBasedConverter<String> {
+ private static final Pattern LINE_BREAK = Pattern.compile("\"[\t\r\n]+\"|'[\t\r\n]+'");
+
@Override
protected String toEscapedString(String value) {
if (value == null) return null;
@@ -38,15 +42,15 @@
if (string == null) return null;
try {
String clean = removeLineBreaksFrom(string);
- return convertFromJavaString(clean.substring(1, clean.length() - 1), true);
+ return convertToJavaString(clean.substring(1, clean.length() - 1), true);
} catch (IllegalArgumentException e) {
throw parsingError(string, node, e);
}
}
private static String removeLineBreaksFrom(String s) {
- if (s == null) return s;
- return s.replaceAll("\"[\t\r\n]+\"|'[\t\r\n]+'", "");
+ if (isEmpty(s)) return s;
+ return LINE_BREAK.matcher(s).replaceAll("");
}
private ValueConverterException parsingError(String string, INode node, Exception cause) {
