diff --git a/docs/about/changelog.md b/docs/about/changelog.md index b57716fd0..417a971cd 100644 --- a/docs/about/changelog.md +++ b/docs/about/changelog.md @@ -8,6 +8,9 @@ priorities and future plans. - Extend CPAN release acceptance coverage to Excel::Writer::XLSX with a timeout suitable for its large test suite. +- Complete `tr///` compatibility for extended Unicode and surrogate scalars, + identity lvalues, and `chop`/`chomp` diagnostics on both execution backends. + - Restore `local` compatibility for tied hash and array elements, sparse arrays, magic stashes, implicit `$_` foreach aliases (including early return), and localized regex captures on both execution backends. diff --git a/src/main/java/org/perlonjava/frontend/parser/OperatorParser.java b/src/main/java/org/perlonjava/frontend/parser/OperatorParser.java index 13ca29bc1..9ec3b439a 100644 --- a/src/main/java/org/perlonjava/frontend/parser/OperatorParser.java +++ b/src/main/java/org/perlonjava/frontend/parser/OperatorParser.java @@ -1461,9 +1461,27 @@ static OperatorNode parseChompChop(Parser parser, LexerToken token, int currentI // stop before any "," at the same precedence level. operand = ListNode.makeList(parser.parseExpression(parser.getPrecedence(",") + 1)); } + if (containsTransliteration(operand)) { + parser.throwError("Can't modify transliteration (tr///) in " + token.text); + } return new OperatorNode(token.text, operand, currentIndex); } + private static boolean containsTransliteration(Node node) { + if (node instanceof OperatorNode operator) { + if (operator.operator.equals("tr") || operator.operator.equals("transliterate")) return true; + return false; + } + if (node instanceof BinaryOperatorNode binary) { + return (binary.operator.equals("=~") || binary.operator.equals("!~")) + && containsTransliteration(binary.right); + } + if (node instanceof ListNode list) { + for (Node element : list.elements) if (containsTransliteration(element)) return true; + } + return false; + } + static OperatorNode parseDieWarn(Parser parser, LexerToken token, int currentIndex) { int dieKeywordIndex = currentIndex; // Capture token position BEFORE parsing args ListNode operand = ListParser.parseZeroOrMoreList(parser, 0, false, true, false, false); diff --git a/src/main/java/org/perlonjava/frontend/parser/ParseInfix.java b/src/main/java/org/perlonjava/frontend/parser/ParseInfix.java index 576d6acaf..bec96f4bc 100644 --- a/src/main/java/org/perlonjava/frontend/parser/ParseInfix.java +++ b/src/main/java/org/perlonjava/frontend/parser/ParseInfix.java @@ -3,11 +3,13 @@ import org.perlonjava.app.cli.CompilerOptions; import org.perlonjava.frontend.analysis.ConstantFoldingVisitor; +import org.perlonjava.frontend.analysis.RegexLiteralAnalyzer; import org.perlonjava.frontend.astnode.*; import org.perlonjava.frontend.lexer.LexerToken; import org.perlonjava.frontend.lexer.LexerTokenType; import org.perlonjava.frontend.semantic.SymbolTable; import org.perlonjava.runtime.operators.WarnDie; +import org.perlonjava.runtime.operators.RuntimeTransliterate; import org.perlonjava.runtime.perlmodule.Strict; import org.perlonjava.runtime.runtimetypes.GlobalVariable; import org.perlonjava.runtime.runtimetypes.ErrorMessageUtil; @@ -661,6 +663,21 @@ private static void rejectAggregateRegexMutation(Parser parser, Node left, || regexOperator.operator.equals("transliterate"))) { return; } + if (!regexOperator.operator.equals("replaceRegex") + && regexOperator.operand instanceof ListNode args + && args.elements.size() >= 3) { + String search = RegexLiteralAnalyzer.constantString(args.elements.get(0)); + String replacement = RegexLiteralAnalyzer.constantString(args.elements.get(1)); + String flags = RegexLiteralAnalyzer.constantString(args.elements.get(2)); + if (search != null && replacement != null && flags != null) { + RuntimeTransliterate operation = new RuntimeTransliterate(); + operation.compileTransliteration(search, replacement, flags); + if (!operation.modifiesTarget()) return; + if (left instanceof StringNode || left instanceof NumberNode) { + parser.throwError("Can't modify constant item in transliteration (tr///)"); + } + } + } if (!(left instanceof OperatorNode aggregate) || !(aggregate.operator.equals("@") || aggregate.operator.equals("%")) || !(aggregate.operand instanceof IdentifierNode identifier)) { @@ -671,7 +688,11 @@ private static void rejectAggregateRegexMutation(Parser parser, Node left, var entry = parser.ctx.symbolTable.getSymbolEntry( aggregate.operator + identifier.name); boolean lexical = entry != null - && ("my".equals(entry.decl()) || "state".equals(entry.decl())); + && ("my".equals(entry.decl()) || "state".equals(entry.decl()) + // eval STRING exposes captured lexicals through internal + // package aliases, without changing their Perl identity. + || (entry.perlPackage() != null + && entry.perlPackage().startsWith("PerlOnJava::_BEGIN_"))); parser.throwError("Can't modify " + (lexical ? "private " : "") + kind + (lexical ? "" : " dereference") + " in " + (regexOperator.operator.equals("replaceRegex") diff --git a/src/main/java/org/perlonjava/runtime/operators/RuntimeTransliterate.java b/src/main/java/org/perlonjava/runtime/operators/RuntimeTransliterate.java index 8d67b4466..bae371285 100644 --- a/src/main/java/org/perlonjava/runtime/operators/RuntimeTransliterate.java +++ b/src/main/java/org/perlonjava/runtime/operators/RuntimeTransliterate.java @@ -17,13 +17,13 @@ public class RuntimeTransliterate { // Mapping from source characters to target characters - private Map translationMap; + private Map translationMap; // Set of characters to delete - private Set deleteSet; + private Set deleteSet; // Set of characters that are part of the search pattern - private Set searchSet; + private Set searchSet; // Modifier flags private boolean complement; @@ -32,7 +32,7 @@ public class RuntimeTransliterate { private boolean returnOriginal; // For complement mode, we need to know the replacement pattern - private List replacementChars; + private List replacementChars; /** * Compiles a RuntimeTransliterate object from a pattern string with optional modifiers. @@ -50,21 +50,20 @@ public RuntimeScalar transliterate(RuntimeScalar originalString, int ctx) { String input = originalString.toString(); StringBuilder result = new StringBuilder(); int count = 0; - Integer lastChar = null; + Long lastChar = null; boolean lastCharWasTransliterated = false; // Track if last char came from transliteration // For complement mode, we need to track replacement index - Map complementMap = new HashMap<>(); + Map complementMap = new HashMap<>(); int replacementIndex = 0; - for (int i = 0; i < input.length(); i++) { - int codePoint = input.codePointAt(i); - - // Handle surrogate pairs for Unicode - only skip if it's a valid supplementary code point - // codePointAt() already combines surrogate pairs, so we just need to skip the second char unit - if (Character.isSupplementaryCodePoint(codePoint)) { - i++; // Skip the low surrogate of a valid surrogate pair - } + for (int i = 0; i < input.length();) { + // PerlUtfString also recognizes the internal representation used for + // surrogate scalars. Treat it as one character; iterating Java UTF-16 + // units here turns \x{d800} into eight unrelated characters. + PerlUtfString.PerlStep step = PerlUtfString.readOnePerlLogical(input, i); + long codePoint = step.codePoint(); + i = step.nextJavaIndex(); boolean matched = false; @@ -93,15 +92,15 @@ public RuntimeScalar transliterate(RuntimeScalar originalString, int ctx) { } lastChar = codePoint; } else { - Integer mappedChar = null; + Long mappedChar = null; // Check if this is the common case of search range 0x00-0xFF if (isRange0x00_0xFF(searchSet)) { // Calculate position relative to first char after range - int position = codePoint - 0x100; + long position = codePoint - 0x100; if (position >= 0) { // Use position as index with wraparound - int index = position % replacementChars.size(); + int index = (int) (position % replacementChars.size()); mappedChar = replacementChars.get(index); } else { // This shouldn't happen for chars matching complement @@ -142,7 +141,7 @@ public RuntimeScalar transliterate(RuntimeScalar originalString, int ctx) { // Character should be deleted - DON'T change lastChar! // We need to preserve it for squashing logic } else if (translationMap.containsKey(codePoint)) { - int mappedChar = translationMap.get(codePoint); + long mappedChar = translationMap.get(codePoint); // Handle squash duplicates - only squash if the last char was also transliterated if (!squashDuplicates || lastChar == null || !lastCharWasTransliterated || !lastChar.equals(mappedChar)) { appendCodePoint(result, mappedChar); @@ -181,14 +180,10 @@ public RuntimeScalar transliterate(RuntimeScalar originalString, int ctx) { return rv; } - // Determine if we need to call set() which will trigger read-only error if applicable - // We must call set() if: - // 1. The string actually changed, OR - // 2. It's an empty string AND we have a replacement operation (not just counting) - boolean hasReplacement = !replacementChars.isEmpty() || deleteUnmatched; - boolean needsSet = !input.equals(resultString) || (input.isEmpty() && hasReplacement); - - if (needsSet) { + // Identity operations count without assigning: assignment would vivify + // missing elements and destroy references even though no change is possible. + // Conversely a modifying operation assigns even when nothing matched. + if (modifiesTarget()) { // Preserve BYTE_STRING type: tr/// on a byte string should produce a byte string boolean wasByteString = originalString.type == RuntimeScalarType.BYTE_STRING; originalString.set(resultString); @@ -201,11 +196,25 @@ public RuntimeScalar transliterate(RuntimeScalar originalString, int ctx) { return new RuntimeScalar(count); } - private boolean isRange0x00_0xFF(Set searchSet) { + /** Whether this compiled operation requires a writable target. */ + public boolean modifiesTarget() { + if (returnOriginal) return false; + if (complement) { + return squashDuplicates || deleteUnmatched || !replacementChars.isEmpty(); + } + if (!deleteSet.isEmpty()) return true; + if (squashDuplicates && !searchSet.isEmpty()) return true; + for (Map.Entry mapping : translationMap.entrySet()) { + if (!mapping.getKey().equals(mapping.getValue())) return true; + } + return false; + } + + private boolean isRange0x00_0xFF(Set searchSet) { // Check if searchSet contains exactly the range 0x00-0xFF if (searchSet.size() != 256) return false; for (int i = 0; i <= 0xFF; i++) { - if (!searchSet.contains(i)) return false; + if (!searchSet.contains((long) i)) return false; } return true; } @@ -221,8 +230,8 @@ public void compileTransliteration(String search, String replace, String modifie returnOriginal = modifiers.contains("r"); // Expand ranges and escapes - List searchChars = expandRangesAndEscapes(search); - List replaceChars = expandRangesAndEscapes(replace); + List searchChars = expandRangesAndEscapes(search); + List replaceChars = expandRangesAndEscapes(replace); // Initialize data structures translationMap = new HashMap<>(); @@ -239,7 +248,7 @@ public void compileTransliteration(String search, String replace, String modifie /** * Sets up the translation map for normal (non-complement) mode. */ - private void setupNormalMapping(List searchChars, List replaceChars) { + private void setupNormalMapping(List searchChars, List replaceChars) { int searchLen = searchChars.size(); int replaceLen = replaceChars.size(); @@ -247,7 +256,7 @@ private void setupNormalMapping(List searchChars, List replace int mappingIndex = 0; for (int i = 0; i < searchLen; i++) { - int searchChar = searchChars.get(i); + long searchChar = searchChars.get(i); // Skip if already mapped (character appeared earlier in search pattern) if (translationMap.containsKey(searchChar) || deleteSet.contains(searchChar)) { @@ -275,13 +284,13 @@ private void setupNormalMapping(List searchChars, List replace * Expands character ranges and escape sequences in the input string. * Returns a list of Unicode code points. */ - private List expandRangesAndEscapes(String input) { - List expanded = new ArrayList<>(); + private List expandRangesAndEscapes(String input) { + List expanded = new ArrayList<>(); int i = 0; while (i < input.length()) { // Parse the current character - List currentChar = new ArrayList<>(); + List currentChar = new ArrayList<>(); int consumed = parseCharAt(input, i, currentChar); if (consumed == 0 || currentChar.isEmpty()) { @@ -295,7 +304,7 @@ private List expandRangesAndEscapes(String input) { nextPos + 1 < input.length()) { // This might be a range - parse the character after the dash - List endChar = new ArrayList<>(); + List endChar = new ArrayList<>(); int endConsumed = parseCharAt(input, nextPos + 1, endChar); if (endConsumed > 0 && !endChar.isEmpty()) { @@ -311,8 +320,8 @@ private List expandRangesAndEscapes(String input) { } // We have a valid range - int start = currentChar.get(0); - int end = endChar.get(0); + long start = currentChar.get(0); + long end = endChar.get(0); // Validate range if (start > end) { @@ -323,7 +332,7 @@ private List expandRangesAndEscapes(String input) { } // Add the range - for (int c = start; c <= end; c++) { + for (long c = start; c <= end; c++) { expanded.add(c); } @@ -345,11 +354,11 @@ private List expandRangesAndEscapes(String input) { * Formats a character for error messages. * Printable characters are shown as-is, non-printable as \x{XXXX}. */ - private String formatCharForError(int codePoint) { + private String formatCharForError(long codePoint) { // Check if the character is printable ASCII or a common printable character // Basic printable ASCII range (excluding control characters) if (codePoint >= 0x20 && codePoint <= 0x7E) { - return new String(Character.toChars(codePoint)); + return new String(Character.toChars((int) codePoint)); } // Format as \x{XXXX} with appropriate padding @@ -364,7 +373,7 @@ private String formatCharForError(int codePoint) { * Parses a character at the given position, handling escape sequences. * Returns the number of characters consumed. */ - private int parseCharAt(String input, int pos, List result) { + private int parseCharAt(String input, int pos, List result) { if (pos >= input.length()) { return 0; } @@ -375,25 +384,25 @@ private int parseCharAt(String input, int pos, List result) { char next = input.charAt(pos + 1); switch (next) { case 'n': - result.add((int) '\n'); + result.add((long) '\n'); return 2; case 't': - result.add((int) '\t'); + result.add((long) '\t'); return 2; case 'r': - result.add((int) '\r'); + result.add((long) '\r'); return 2; case 'f': - result.add((int) '\f'); + result.add((long) '\f'); return 2; case 'b': - result.add((int) '\b'); + result.add((long) '\b'); return 2; case 'a': - result.add(0x07); + result.add(0x07L); return 2; // Bell character case 'e': - result.add(0x1B); + result.add(0x1BL); return 2; // Escape character case '0': case '1': @@ -407,9 +416,18 @@ private int parseCharAt(String input, int pos, List result) { return 1 + parseOctalSequence(input, pos + 1, result); case 'x': return 2 + parseHexSequence(input, pos + 2, result); + case 'c': + // Transliteration source keeps escape syntax intact so + // ranges can be expanded here. Perl's \cX is X xor 0x40. + if (pos + 2 < input.length()) { + result.add((long) (input.charAt(pos + 2) ^ 0x40)); + return 3; + } + result.add((long) 'c'); + return 2; case '-': // Escaped dash - result.add((int) '-'); + result.add((long) '-'); return 2; case 'N': if (pos + 2 < input.length() && input.charAt(pos + 2) == '{') { @@ -425,7 +443,7 @@ private int parseCharAt(String input, int pos, List result) { // Try to resolve the Unicode character name try { int codePoint = UnicodeResolver.getCodePointFromName(content); - result.add(codePoint); + result.add((long) codePoint); return closePos - pos + 1; } catch (IllegalArgumentException e) { // Check if it's a named sequence (multi-character) @@ -446,17 +464,18 @@ private int parseCharAt(String input, int pos, List result) { throw new RuntimeException("Unknown charname ''"); } } - result.add((int) 'N'); + result.add((long) 'N'); return 2; default: // Other escaped character - result.add((int) next); + result.add((long) next); return 2; } } else { // Regular character - result.add((int) ch); - return 1; + PerlUtfString.PerlStep step = PerlUtfString.readOnePerlLogical(input, pos); + result.add(step.codePoint()); + return step.nextJavaIndex() - pos; } } @@ -464,7 +483,7 @@ private int parseCharAt(String input, int pos, List result) { * Parses octal escape sequences (\0, \77, \377, etc.). * Returns the number of characters consumed (not including the initial backslash). */ - private int parseOctalSequence(String input, int start, List result) { + private int parseOctalSequence(String input, int start, List result) { int value = 0; int digits = 0; int pos = start; @@ -487,7 +506,7 @@ private int parseOctalSequence(String input, int start, List result) { value = 0377; } - result.add(value); + result.add((long) value); return digits; } @@ -495,9 +514,9 @@ private int parseOctalSequence(String input, int start, List result) { * Parses hexadecimal escape sequences (\xNN or \x{NNNN}). * Returns the number of additional characters consumed (after \x). */ - private int parseHexSequence(String input, int start, List result) { + private int parseHexSequence(String input, int start, List result) { if (start >= input.length()) { - result.add((int) 'x'); // Invalid sequence, treat as literal 'x' + result.add((long) 'x'); // Invalid sequence, treat as literal 'x' return -2; // Back up to just after '\' } @@ -509,7 +528,7 @@ private int parseHexSequence(String input, int start, List result) { if (isValidHexString(hexStr)) { try { int value = Integer.parseInt(hexStr, 16); - result.add(value); + result.add((long) value); return end - start + 1; // Consumed {NNNN} } catch (NumberFormatException e) { // Fall through to error case @@ -523,13 +542,13 @@ private int parseHexSequence(String input, int start, List result) { isHexDigit(input.charAt(start + 1))) { String hexStr = input.substring(start, start + 2); int value = Integer.parseInt(hexStr, 16); - result.add(value); + result.add((long) value); return 2; // Consumed NN } } // Invalid sequence - treat \x as literal characters - result.add((int) 'x'); + result.add((long) 'x'); return -2; // Back up to just after '\' } @@ -556,7 +575,13 @@ private boolean isHexDigit(char ch) { /** * Appends a Unicode code point to the StringBuilder. */ - private void appendCodePoint(StringBuilder sb, int codePoint) { - sb.appendCodePoint(codePoint); + private void appendCodePoint(StringBuilder sb, long codePoint) { + if (codePoint > 0x10FFFFL) { + sb.append(PerlUtfString.encodeBeyondUnicode(codePoint)); + } else if (codePoint >= 0xD800L && codePoint <= 0xDFFFL) { + sb.append(PerlUtfString.encodeSurrogate(codePoint)); + } else { + sb.appendCodePoint((int) codePoint); + } } } diff --git a/src/test/resources/unit/transliteration_identity_lvalues.t b/src/test/resources/unit/transliteration_identity_lvalues.t new file mode 100644 index 000000000..8bdba2017 --- /dev/null +++ b/src/test/resources/unit/transliteration_identity_lvalues.t @@ -0,0 +1,21 @@ +use strict; +use warnings; +use Test::More; + +my @array; +my %hash; +eval '$array[-1] =~ tr/N/N/'; +is($@, '', 'identity transliteration permits a missing negative array element'); +eval '$array[1] =~ tr/N/N/'; +is($@, '', 'identity transliteration permits a missing positive array element'); +is(scalar @array, 0, 'identity transliteration does not extend an array'); +eval '$hash{missing} =~ tr/N/N/'; +is($@, '', 'identity transliteration permits a missing hash element'); +is(scalar keys %hash, 0, 'identity transliteration does not insert a hash key'); +my $reference = \ 'NN'; +my $count = $reference =~ tr/N/N/; +is(ref $reference, 'SCALAR', 'identity transliteration preserves references'); +my $value = 'NN'; +is($value =~ tr/N/N/, 2, 'identity transliteration still counts matches'); +is($value, 'NN', 'identity transliteration preserves its input'); +done_testing(); diff --git a/src/test/resources/unit/transliteration_lvalue_diagnostics.t b/src/test/resources/unit/transliteration_lvalue_diagnostics.t new file mode 100644 index 000000000..676461c25 --- /dev/null +++ b/src/test/resources/unit/transliteration_lvalue_diagnostics.t @@ -0,0 +1,27 @@ +use strict; +use warnings; +use Test::More; + +eval q{'123' =~ tr/1/2/}; +like($@, qr/^Can't modify constant item in transliteration \(tr\/\/\/\)/, + 'modifying transliteration rejects a constant'); +eval q{'123' =~ tr/1/1/}; +is($@, '', 'identity transliteration accepts a constant'); +my ($s, @a); +for my $op ('chop', 'chomp') { + for my $bind ('', '$s =~ ', '@a =~ ') { + for my $replacement ('a', 'b') { + for my $flags ('', 'r') { + local $SIG{__WARN__} = sub {}; + eval "$op(${bind}tr/a/$replacement/$flags)"; + my $expected = $bind eq '@a =~ ' && $replacement eq 'b' && !$flags + ? 'private array in transliteration (tr///)' + : "transliteration (tr///) in $op"; + like($@, qr/^Can't modify \Q$expected\E/, 'result and target lvalue diagnostics'); + } + } + } +} +eval q{my $x; my $y = 'a'; chop($x = ($y =~ tr/a/b/))}; +is($@, '', 'assignment containing transliteration is still an lvalue'); +done_testing(); diff --git a/src/test/resources/unit/transliteration_unicode.t b/src/test/resources/unit/transliteration_unicode.t new file mode 100644 index 000000000..99c0b4723 --- /dev/null +++ b/src/test/resources/unit/transliteration_unicode.t @@ -0,0 +1,22 @@ +use strict; +use warnings; +use utf8; +use Test::More; + +no warnings 'utf8'; +my $surrogate = "\x{d800}\x{ffff}"; +$surrogate =~ tr/\0/A/c; +is($surrogate, 'AA', 'complement transliterates surrogate scalars as single characters'); + +my $extended = "A\x{ffff}B"; +$extended =~ tr/\x{ffff}/\x{1ffff}/; +is($extended, "A\x{1ffff}B", 'extended Unicode code points survive transliteration'); + +my $beyond_unicode = 'cb'; +$beyond_unicode =~ tr{aabc}{d\x{d0000}}; +is($beyond_unicode, "\x{d0000}\x{d0000}", 'replacement scalars above Unicode remain logical characters'); + +my $named = "\x{cb}"; +$named =~ tr[\N{U+00CB}\N{U+00EB}\N{U+2010}][\N{U+0401}\N{U+0451}\-]; +is($named, "\x{401}", 'Unicode names are expanded in transliteration lists'); +done_testing();