Fix JavaScript extraction of escaped Unicode surrogate pairs

babel/messages/extract.py

@@ -849,7 +849,12 @@ def extract_javascript(
             elif token.type in ('string', 'template_string'):
                 new_value = unquote_string(token.value)
                 if concatenate_next:
-                    last_argument = (last_argument or '') + new_value
+                    # A surrogate pair may span multiple string literals.
+                    last_argument = (
+                        ((last_argument or '') + new_value)
+                        .encode('utf-16-le', 'surrogatepass')
+                        .decode('utf-16-le', 'surrogatepass')
+                    )
                     concatenate_next = False
                 else:
                     last_argument = new_value

babel/messages/jslexer.py

@@ -161,7 +161,8 @@ def unquote_string(string: str) -> str:
     if pos < len(string):
         add(string[pos:])
 
-    return ''.join(result)
+    # Combine UTF-16 surrogate pairs while preserving unmatched surrogates.
+    return ''.join(result).encode('utf-16-le', 'surrogatepass').decode('utf-16-le', 'surrogatepass')
 
 
 def tokenize(

tests/messages/test_js_extract.py

@@ -20,6 +20,56 @@ def test_simple_extract():
                         (3, ('s', 'p'), [], None)]
 
 
+@pytest.mark.parametrize('source', [
+    r'gettext("\uD83D\uDE00")',
+    r"gettext('\uD83D\uDE00')",
+    r'gettext(`\uD83D\uDE00`)',
+    r'gettext`\uD83D\uDE00`',
+    'gettext("\U0001f600")',
+    r'gettext("\uD83D" + "\uDE00")',
+    r"gettext('\uD83D' + '\uDE00')",
+    r'gettext(`\uD83D` + `\uDE00`)',
+    r'gettext("\uD83D" + `\uDE00`)',
+    r'gettext("\uD83D" + "" + "\uDE00")',
+])
+def test_extract_surrogate_pair(source):
+    messages = list(extract.extract('javascript', BytesIO(source.encode('utf-8'))))
+
+    assert messages == [(1, '\U0001f600', [], None)]
+
+
+@pytest.mark.parametrize(('source', 'expected'), [
+    (r'gettext("\uD83D" + "-" + "\uDE00")', '\ud83d-\ude00'),
+    (r'gettext("\uDE00" + "\uD83D")', '\ude00\ud83d'),
+    (r'gettext("\\uD83D" + "\\uDE00")', r'\uD83D\uDE00'),
+    (r'gettext("\uD83D" + "\\uDE00")', '\ud83d' + r'\uDE00'),
+])
+def test_extract_concatenated_surrogate_boundaries(source, expected):
+    messages = list(extract.extract('javascript', BytesIO(source.encode('utf-8'))))
+
+    assert messages == [(1, expected, [], None)]
+
+
+@pytest.mark.parametrize(('source', 'message', 'context'), [
+    (
+        r'ngettext("\uD83D" + "\uDE00", "\uD83D" + "\uDE03", n)',
+        ('\U0001f600', '\U0001f603'),
+        None,
+    ),
+    (r'ngettext("\uD83D", "\uDE00", n)', ('\ud83d', '\ude00'), None),
+    (
+        r'pgettext("\uD83D" + "\uDE00", "\uD83D" + "\uDE03")',
+        '\U0001f603',
+        '\U0001f600',
+    ),
+    (r'pgettext("\uD83D", "\uDE00")', '\ude00', '\ud83d'),
+])
+def test_extract_surrogate_pair_arguments(source, message, context):
+    messages = list(extract.extract('javascript', BytesIO(source.encode('utf-8'))))
+
+    assert messages == [(1, message, [], context)]
+
+
 def test_various_calls():
     buf = BytesIO(b"""\
 msg1 = _(i18n_arg.replace(/"/, '"'))

tests/messages/test_jslexer.py

@@ -1,3 +1,5 @@
+import pytest
+
 from babel.messages import jslexer
 
 
@@ -8,6 +10,25 @@ def test_unquote():
     assert jslexer.unquote_string(r'"\xebb"') == "ëb"
 
 
+@pytest.mark.parametrize(('string', 'expected'), [
+    (r'"\uD83D\uDE00"', '\U0001f600'),
+    (r'"before \ud83d\ude00 after"', 'before \U0001f600 after'),
+    (r'"\uD800\uDC00\uDBFF\uDFFF"', '\U00010000\U0010ffff'),
+    (r'"\uD83D"', '\ud83d'),
+    (r'"\uDE00"', '\ude00'),
+    (r'"\uDE00\uD83D"', '\ude00\ud83d'),
+    (r'"\uD83D\uD83D\uDE00"', '\ud83d\U0001f600'),
+    (r'"\uD83D-\uDE00"', '\ud83d-\ude00'),
+    (r'"\uD83D\n\uDE00"', '\ud83d\n\ude00'),
+    (r'"\\uD83D\\uDE00"', r'\uD83D\uDE00'),
+    (r'"\uD83D\\uDE00"', '\ud83d' + r'\uDE00'),
+    (r'"\uFEFF\u00EB\uFFFF"', '\ufeffë\uffff'),
+    ('"\U0001f600"', '\U0001f600'),
+])
+def test_unquote_surrogates(string, expected):
+    assert jslexer.unquote_string(string) == expected
+
+
 def test_dollar_in_identifier():
     assert list(jslexer.tokenize('dollar$dollar')) == [('name', 'dollar$dollar', 1)]