diff --git a/TypeScript/README.md b/TypeScript/README.md index 3a626b3..699071e 100644 --- a/TypeScript/README.md +++ b/TypeScript/README.md @@ -220,6 +220,51 @@ Replaces referenced variable names. The replacement record is keyed by variable Replaces a scope qualifier for variables known to the interpreter state. +### `rewriteLegacyVariableReferences(code, replacements, options?)` + +Safely rewrites references whose stored variable names are not valid Arcscript +identifiers. Replacements are keyed by variable ID. The method accepts raw +Arcscript code and preserves its whitespace, entities, strings, and mentions. + +String literals, mention spans, operators, and identifiers are recognized with +the generated Arcscript lexer, and potential Arcscript expressions are checked +with the generated parser. Accepted references are replaced directly at their +original source ranges. + +```ts +const interpreter = new Interpreter({ + state: { + score: { + id: 'score', + name: 'score value', + type: 'integer', + defaultValue: 0, + scope: 'hero', + }, + }, +}); + +const result = interpreter.rewriteLegacyVariableReferences( + 'hero . score value += 1', + { score: 'score_value' }, + { scopeNames: ['hero'] } +); + +console.log(result.code); +// hero . score_value += 1 +console.log(result.blockedVariableIds); +// [] +``` + +`scopeNames` should contain every board/component scope name, including +duplicates. Variables are reported in `blockedVariableIds` and left unchanged +when their references could instead be Arcscript syntax, when their name is +duplicated within a scope, when a scoped qualifier is invalid or ambiguous, or +when the requested replacement is not a non-reserved Arcscript identifier. +All accepted replacements are located in the original source before any are +applied, so overlapping names cannot consume each other and replacement text +is never reinterpreted. + ## Development This project uses the version of pnpm declared in `package.json`. @@ -228,6 +273,7 @@ This project uses the version of pnpm declared in `package.json`. pnpm install pnpm build pnpm test +pnpm lint ``` The build writes compiled files and declarations to `dist/`. diff --git a/TypeScript/package.json b/TypeScript/package.json index 48cb24c..9057182 100644 --- a/TypeScript/package.json +++ b/TypeScript/package.json @@ -2,11 +2,13 @@ "name": "@arcweave/arcscript", "version": "3.0.0", "description": "An interpreter for arcscript using antlr4", + "type": "module", "main": "dist/index.js", "types": "dist/index.d.ts", "scripts": { "build": "tsc --build", "clean": "rm -rf dist", + "lint": "eslint .", "test": "vitest" }, "exports": { @@ -52,6 +54,7 @@ "dependencies": { "antlr4": "4.13.2", "bignumber.js": "^9.3.1", + "entities": "^7.0.1", "lodash.clonedeep": "^4.5.0" } } diff --git a/TypeScript/pnpm-lock.yaml b/TypeScript/pnpm-lock.yaml index ba7bd33..613dbd6 100644 --- a/TypeScript/pnpm-lock.yaml +++ b/TypeScript/pnpm-lock.yaml @@ -14,6 +14,9 @@ importers: bignumber.js: specifier: ^9.3.1 version: 9.3.1 + entities: + specifier: ^7.0.1 + version: 7.0.1 lodash.clonedeep: specifier: ^4.5.0 version: 4.5.0 diff --git a/TypeScript/src/LegacyVariableReferenceRewriter.ts b/TypeScript/src/LegacyVariableReferenceRewriter.ts new file mode 100644 index 0000000..24de4be --- /dev/null +++ b/TypeScript/src/LegacyVariableReferenceRewriter.ts @@ -0,0 +1,803 @@ +import { DecodingMode, EntityDecoder, htmlDecodeTree } from 'entities/decode'; +import ArcscriptLexer from './Generated/ArcscriptLexer.js'; +import type { + ArcscriptStateDef, + LegacyVariableRewriteOptions, + LegacyVariableRewriteResult, +} from './types.js'; +import { isGlobalScope } from './scope.js'; +import { CharStream } from 'antlr4'; + +type SourceRange = { + start: number; + end: number; +}; + +type DecodedSource = { + text: string; + sourceUnits: SourceRange[]; +}; + +type ProtectedRanges = { + all: SourceRange[]; + strings: SourceRange[]; +}; + +type Candidate = { + id: string; + oldName: string; + newName: string; + scope: string | null; +}; + +type CandidateMatch = { + candidate: Candidate; + start: number; + end: number; + replacementStart: number; + replacementEnd: number; +}; + +type SourceReplacement = { + start: number; + end: number; + text: string; +}; + +const CALLABLE_RESERVED_NAMES = new Set([ + 'sqr', + 'sqrt', + 'abs', + 'random', + 'roll', + 'show', + 'reset', + 'resetAll', + 'resetVisits', + 'min', + 'max', + 'round', + 'visits', +]); + +const RESERVED_NAMES = new Set([ + ...CALLABLE_RESERVED_NAMES, + 'if', + 'endif', + 'else', + 'elseif', + 'and', + 'or', + 'not', + 'is', + 'true', + 'false', +]); + +const OPERATOR_TOKEN_NAMES = new Set([ + 'LPAREN', + 'RPAREN', + 'ASSIGNMUL', + 'ASSIGNDIV', + 'ASSIGNADD', + 'ASSIGNSUB', + 'ASSIGNMOD', + 'MUL', + 'DIV', + 'ADD', + 'SUB', + 'MOD', + 'GE', + 'GT', + 'LE', + 'LT', + 'EQ', + 'NE', + 'AND', + 'OR', + 'ASSIGN', + 'NEG', + 'COMMA', + 'LBRACE', + 'RBRACE', + 'IFKEYWORD', + 'ELSEKEYWORD', + 'ELSEIFKEYWORD', + 'ENDIFKEYWORD', + 'ANDKEYWORD', + 'ORKEYWORD', + 'ISKEYWORD', + 'NOTKEYWORD', +]); + +const DELIMITER_TOKEN_NAMES = new Set([ + 'LPAREN', + 'RPAREN', + 'COMMA', + 'LBRACE', + 'RBRACE', +]); + +const EXPRESSION_KEYWORDS_BEFORE = new Set([ + 'IFKEYWORD', + 'ELSEIFKEYWORD', + 'ANDKEYWORD', + 'ORKEYWORD', + 'ISKEYWORD', + 'NOTKEYWORD', +]); + +const EXPRESSION_KEYWORDS_AFTER = new Set([ + 'ANDKEYWORD', + 'ORKEYWORD', + 'ISKEYWORD', +]); + +const IDENTIFIER_PART_PATTERN = /^[\p{L}\p{N}\p{M}_$]$/u; + +export default class LegacyVariableReferenceRewriter { + constructor( + private readonly state: ArcscriptStateDef, + private readonly parsesAsArcscript: (code: string) => boolean + ) {} + + rewrite( + code: string, + replacementsByVariableId: Record, + options: LegacyVariableRewriteOptions = {} + ): LegacyVariableRewriteResult { + const candidates = Object.values(this.state) + .filter(variable => Object.hasOwn(replacementsByVariableId, variable.id)) + .map(variable => ({ + id: variable.id, + oldName: variable.name, + newName: replacementsByVariableId[variable.id], + scope: isGlobalScope(variable.scope) + ? null + : (variable.scope as string), + })); + const blocked = this.inherentlyBlockedCandidates(candidates, options); + const decoded = this.decodeSource(code); + const possibleMatches = candidates + .filter(candidate => !blocked.has(candidate.id)) + .flatMap(candidate => this.candidateMatches(decoded.text, candidate)); + const sourceMatches = possibleMatches.map(match => + this.mapMatchToSource(match, decoded, code.length) + ); + const decodedProtectedRanges = this.protectedRanges( + decoded.text, + possibleMatches + ); + const sourceProtectedRanges = this.protectedRanges( + code, + sourceMatches, + false, + decodedProtectedRanges.strings.map(range => + this.mapStringInteriorToSource(range, decoded, code.length) + ) + ); + const matches = this.nonOverlappingMatches( + possibleMatches.filter( + (match, index) => + !decodedProtectedRanges.all.some(range => + this.rangesOverlap(range, match) + ) && + !sourceProtectedRanges.all.some(range => + this.rangesOverlap(range, sourceMatches[index]) + ) + ) + ).map(match => this.mapMatchToSource(match, decoded, code.length)); + + const matchesByCandidate = new Map(); + for (const match of matches) { + const candidateMatches = matchesByCandidate.get(match.candidate.id) ?? []; + candidateMatches.push(match); + matchesByCandidate.set(match.candidate.id, candidateMatches); + } + + for (const candidate of candidates) { + if ( + (matchesByCandidate.get(candidate.id)?.length ?? 0) > 0 && + this.isAmbiguous(candidate, options) + ) { + blocked.add(candidate.id); + } + } + + const sourceReplacements = matches + .filter(match => !blocked.has(match.candidate.id)) + .map(match => ({ + start: match.replacementStart, + end: match.replacementEnd, + text: match.candidate.newName, + })); + + return { + code: this.applyReplacements(code, sourceReplacements), + blockedVariableIds: [...blocked].sort(), + }; + } + + private inherentlyBlockedCandidates( + candidates: Candidate[], + options: LegacyVariableRewriteOptions + ) { + const blocked = new Set(); + const scopeNames = options.scopeNames ?? [ + ...new Set( + Object.values(this.state) + .map(variable => variable.scope) + .filter((scope): scope is string => !isGlobalScope(scope)) + ), + ]; + const scopeCounts = new Map(); + for (const scope of scopeNames) { + scopeCounts.set(scope, (scopeCounts.get(scope) ?? 0) + 1); + } + + for (const candidate of candidates) { + if ( + candidate.oldName.length === 0 || + this.hasUnescapedDoubleQuote(candidate.oldName) || + this.hasSingleQuoteDelimiter(candidate.oldName) || + !this.isValidIdentifier(candidate.newName) || + RESERVED_NAMES.has(candidate.newName) || + (candidate.scope !== null && + (!this.isValidIdentifier(candidate.scope) || + RESERVED_NAMES.has(candidate.scope) || + (scopeCounts.get(candidate.scope) ?? 0) > 1)) + ) { + blocked.add(candidate.id); + } + } + + const duplicateNames = new Map(); + const candidateIds = new Set(candidates.map(candidate => candidate.id)); + for (const variable of Object.values(this.state)) { + const scope = isGlobalScope(variable.scope) + ? '' + : (variable.scope as string); + const key = `${scope}\0${variable.name}`; + const ids = duplicateNames.get(key) ?? []; + ids.push(variable.id); + duplicateNames.set(key, ids); + } + for (const ids of duplicateNames.values()) { + if (ids.length > 1) { + ids.filter(id => candidateIds.has(id)).forEach(id => blocked.add(id)); + } + } + + return blocked; + } + + private candidateMatches(text: string, candidate: Candidate) { + const matches: CandidateMatch[] = []; + if (candidate.scope === null) { + let start = text.indexOf(candidate.oldName); + while (start !== -1) { + const end = start + candidate.oldName.length; + if ( + this.hasIdentifierBoundaries(text, start, end) && + this.hasWhitespaceNameBoundaries( + text, + start, + end, + candidate.oldName + ) && + !this.isMemberExpressionPart(text, start, end) && + !this.isCallableInvocation(text, end, candidate.oldName) + ) { + matches.push({ + candidate, + start, + end, + replacementStart: start, + replacementEnd: end, + }); + } + start = text.indexOf(candidate.oldName, start + 1); + } + + return matches; + } + + let scopeStart = text.indexOf(candidate.scope); + while (scopeStart !== -1) { + let position = scopeStart + candidate.scope.length; + while (this.isWhitespace(text[position])) position += 1; + if (text[position] === '.') { + position += 1; + while (this.isWhitespace(text[position])) position += 1; + if (text.startsWith(candidate.oldName, position)) { + const end = position + candidate.oldName.length; + if ( + this.hasIdentifierBoundaries(text, scopeStart, end) && + this.hasWhitespaceNameBoundaries( + text, + position, + end, + candidate.oldName + ) + ) { + matches.push({ + candidate, + start: scopeStart, + end, + replacementStart: position, + replacementEnd: end, + }); + } + } + } + scopeStart = text.indexOf(candidate.scope, scopeStart + 1); + } + + return matches; + } + + private nonOverlappingMatches(matches: CandidateMatch[]) { + const byStart = new Map(); + for (const match of matches) { + const matchesAtStart = byStart.get(match.start) ?? []; + matchesAtStart.push(match); + matchesAtStart.sort( + (left, right) => + right.end - right.start - (left.end - left.start) || + left.candidate.id.localeCompare(right.candidate.id) + ); + byStart.set(match.start, matchesAtStart); + } + + const outside: CandidateMatch[] = []; + let position = Math.min(...byStart.keys()); + const lastPosition = Math.max(...byStart.keys()); + while (position <= lastPosition) { + const match = byStart.get(position)?.[0]; + if (match) { + outside.push(match); + position = match.end; + continue; + } + position += 1; + } + + return outside; + } + + private isAmbiguous( + candidate: Candidate, + options: LegacyVariableRewriteOptions + ) { + const name = candidate.oldName.trim(); + const tokens = this.tokensForCode(name); + if ( + tokens.tokenNames[0] === 'INTEGER' || + tokens.tokenNames[0] === 'FLOAT' + ) { + return true; + } + if ( + name !== candidate.oldName && + (RESERVED_NAMES.has(name) || + this.parsesAsArcscript( + candidate.scope === null ? name : `${candidate.scope}.${name}` + )) + ) { + return true; + } + if (RESERVED_NAMES.has(candidate.oldName)) { + return !CALLABLE_RESERVED_NAMES.has(candidate.oldName); + } + if (candidate.scope === null && candidate.oldName.includes('.')) { + const scopeNames = new Set( + options.scopeNames ?? + Object.values(this.state) + .map(variable => variable.scope) + .filter((value): value is string => !isGlobalScope(value)) + ); + const dotIndex = tokens.tokenNames.indexOf('DOT'); + const scope = dotIndex > 0 ? tokens.tokens[dotIndex - 1].text : ''; + if (scope.length === 0 || scopeNames.has(scope)) { + return true; + } + } + + const tokenNames = tokens.tokenNames; + const hasOperator = + tokenNames.some(name => OPERATOR_TOKEN_NAMES.has(name)) || + tokens.tokens.some( + (token, index) => + tokenNames[index] === 'LEGACY_CHARACTER' && + '-+*/%<>=!&|(),{}'.includes(token.text) + ); + if (!hasOperator) { + return false; + } + if (tokenNames.some(name => DELIMITER_TOKEN_NAMES.has(name))) { + return true; + } + if ( + this.isOperatorToken(tokens, 0) || + this.isOperatorToken(tokens, tokens.tokens.length - 1) + ) { + return true; + } + + for (const other of Object.values(this.state)) { + if ( + other.id !== candidate.id && + other.name.length > 0 && + this.containsLegacyOperand(candidate.oldName, other.name) + ) { + return true; + } + } + + return this.parsesAsArcscript( + candidate.scope === null + ? candidate.oldName + : `${candidate.scope}.${candidate.oldName}` + ); + } + + private containsLegacyOperand(compoundName: string, legacyName: string) { + let position = compoundName.indexOf(legacyName); + while (position !== -1) { + const end = position + legacyName.length; + if (this.hasIdentifierBoundaries(compoundName, position, end)) { + const outside = + compoundName.slice(0, position) + compoundName.slice(end); + const tokens = this.tokensForCode(outside); + if ( + tokens.tokenNames.some(name => OPERATOR_TOKEN_NAMES.has(name)) || + tokens.tokens.some( + (token, index) => + tokens.tokenNames[index] === 'LEGACY_CHARACTER' && + '-+*/%<>=!&|(),{}'.includes(token.text) + ) + ) { + return true; + } + } + position = compoundName.indexOf(legacyName, position + 1); + } + + return false; + } + + private tokensForCode(code: string) { + const wrapped = `
${code}
`; + const lexer = this.createLenientLexer(wrapped); + const tokenTypeNames = lexer.getSymbolicNames(); + const tokens = lexer + .getAllTokens() + .filter( + token => + token.type !== ArcscriptLexer.CODESTART && + token.type !== ArcscriptLexer.CODEEND + ); + + return { + tokens, + tokenNames: tokens.map(token => tokenTypeNames[token.type]), + }; + } + + private isOperatorToken( + tokens: ReturnType, + index: number + ) { + return ( + OPERATOR_TOKEN_NAMES.has(tokens.tokenNames[index]) || + (tokens.tokenNames[index] === 'LEGACY_CHARACTER' && + '-+*/%<>=!&|(),{}'.includes(tokens.tokens[index].text)) + ); + } + + private protectedRanges( + code: string, + matches: CandidateMatch[], + protectUnmatchedQuotes = true, + neutralQuoteRanges: SourceRange[] = [] + ): ProtectedRanges { + const prefix = '
';
+    const maskedCode = this.maskLegacyMatches(code, matches).split('');
+    for (const range of neutralQuoteRanges) {
+      for (let position = range.start; position < range.end; position += 1) {
+        if (maskedCode[position] === '"' || maskedCode[position] === "'") {
+          maskedCode[position] = this.isEscapedCharacter(code, position)
+            ? 'n'
+            : 'x';
+        }
+      }
+    }
+    const source = `${prefix}${maskedCode.join('')}
`; + const tokens = this.createLenientLexer(source).getAllTokens(); + const ranges: SourceRange[] = []; + const stringRanges: SourceRange[] = []; + let mentionStart: number | null = null; + for (const token of tokens) { + const start = token.start - prefix.length; + const end = token.stop + 1 - prefix.length; + if (token.type === ArcscriptLexer.MENTION_TAG_OPEN) { + mentionStart = start; + } else if ( + token.type === ArcscriptLexer.MENTION_TAG_CLOSE && + mentionStart !== null + ) { + ranges.push({ + start: Math.max(0, mentionStart), + end: Math.max(0, end), + }); + mentionStart = null; + } else if (token.type === ArcscriptLexer.STRING) { + const range = { + start: Math.max(0, start), + end: Math.max(0, end), + }; + ranges.push(range); + stringRanges.push(range); + } else if ( + protectUnmatchedQuotes && + token.type === ArcscriptLexer.LEGACY_CHARACTER && + (token.text === '"' || token.text === "'") + ) { + ranges.push({ start: Math.max(0, start), end: code.length }); + } + } + + return { all: ranges, strings: stringRanges }; + } + + private maskLegacyMatches(code: string, matches: CandidateMatch[]) { + const masked = code.split(''); + for (const match of this.nonOverlappingMatches(matches)) { + const tokens = this.tokensForCode(match.candidate.oldName); + if ( + match.candidate.oldName.startsWith('"') || + match.candidate.oldName.startsWith("'") || + tokens.tokenNames.includes('STRING') + ) { + continue; + } + + for (let position = match.start; position < match.end; position += 1) { + if (masked[position] !== '\\') { + masked[position] = this.isEscapedCharacter(code, position) + ? 'n' + : 'x'; + } + } + } + + return masked.join(''); + } + + private createLenientLexer(code: string) { + const lexer = new ArcscriptLexer(new CharStream(code)); + lexer.removeErrorListeners(); + return lexer; + } + + private decodeSource(source: string): DecodedSource { + let text = ''; + const sourceUnits: SourceRange[] = []; + let position = 0; + while (position < source.length) { + if (source[position] === '&') { + let decodedEntity = ''; + const decoder = new EntityDecoder(htmlDecodeTree, codePoint => { + decodedEntity += String.fromCodePoint(codePoint); + }); + decoder.startEntity(DecodingMode.Strict); + const consumed = decoder.write(source, position + 1); + if (consumed > 0) { + text += decodedEntity; + for (let index = 0; index < decodedEntity.length; index += 1) { + sourceUnits.push({ + start: position, + end: position + consumed, + }); + } + position += consumed; + continue; + } + } + + const character = String.fromCodePoint(source.codePointAt(position)!); + text += character; + for (let index = 0; index < character.length; index += 1) { + sourceUnits.push({ + start: position, + end: position + character.length, + }); + } + position += character.length; + } + + return { text, sourceUnits }; + } + + private mapMatchToSource( + match: CandidateMatch, + decoded: DecodedSource, + sourceLength: number + ): CandidateMatch { + return { + ...match, + start: decoded.sourceUnits[match.start]?.start ?? sourceLength, + end: decoded.sourceUnits[match.end - 1]?.end ?? sourceLength, + replacementStart: + decoded.sourceUnits[match.replacementStart]?.start ?? sourceLength, + replacementEnd: + decoded.sourceUnits[match.replacementEnd - 1]?.end ?? sourceLength, + }; + } + + private mapStringInteriorToSource( + range: SourceRange, + decoded: DecodedSource, + sourceLength: number + ): SourceRange { + return { + start: decoded.sourceUnits[range.start]?.end ?? sourceLength, + end: decoded.sourceUnits[range.end - 1]?.start ?? sourceLength, + }; + } + + private hasUnescapedDoubleQuote(text: string) { + let position = text.indexOf('"'); + while (position !== -1) { + if (!this.isEscapedCharacter(text, position)) { + return true; + } + position = text.indexOf('"', position + 1); + } + + return false; + } + + private hasSingleQuoteDelimiter(text: string) { + let position = text.indexOf("'"); + while (position !== -1) { + if (!this.isEscapedCharacter(text, position)) { + const prefixTokens = this.tokensForCode(text.slice(0, position)); + const previousToken = prefixTokens.tokenNames.at(-1); + if ( + !this.isIdentifierPart(this.codePointBefore(text, position)) || + (previousToken !== 'IDENTIFIER' && + previousToken !== 'LEGACY_CHARACTER') + ) { + return true; + } + } + position = text.indexOf("'", position + 1); + } + + return false; + } + + private hasIdentifierBoundaries(text: string, start: number, end: number) { + const previous = this.codePointBefore(text, start); + const next = String.fromCodePoint(text.codePointAt(end) ?? 0); + return !this.isIdentifierPart(previous) && !this.isIdentifierPart(next); + } + + private hasWhitespaceNameBoundaries( + text: string, + start: number, + end: number, + name: string + ) { + if (![...name].some(character => this.isWhitespace(character))) { + return true; + } + + let before = start - 1; + while (before >= 0 && this.isWhitespace(text[before])) before -= 1; + let after = end; + while (after < text.length && this.isWhitespace(text[after])) after += 1; + + return ( + this.hasExpressionBoundaryBefore(text, before) && + this.hasExpressionBoundaryAfter(text, after) + ); + } + + private hasExpressionBoundaryBefore(text: string, end: number) { + if (!this.isIdentifierPart(this.codePointBefore(text, end + 1))) { + return true; + } + + let start = end; + while (start >= 0 && this.isIdentifierPart(text[start])) start -= 1; + const tokenName = this.tokensForCode(text.slice(start + 1, end + 1)) + .tokenNames[0]; + + return EXPRESSION_KEYWORDS_BEFORE.has(tokenName); + } + + private hasExpressionBoundaryAfter(text: string, start: number) { + if ( + !this.isIdentifierPart(String.fromCodePoint(text.codePointAt(start) ?? 0)) + ) { + return true; + } + + let end = start; + while (end < text.length && this.isIdentifierPart(text[end])) end += 1; + const tokenName = this.tokensForCode(text.slice(start, end)).tokenNames[0]; + + return EXPRESSION_KEYWORDS_AFTER.has(tokenName); + } + + private isMemberExpressionPart(text: string, start: number, end: number) { + let before = start - 1; + while (before >= 0 && this.isWhitespace(text[before])) before -= 1; + let after = end; + while (after < text.length && this.isWhitespace(text[after])) after += 1; + return text[before] === '.' || text[after] === '.'; + } + + private isCallableInvocation(text: string, end: number, name: string) { + if (!CALLABLE_RESERVED_NAMES.has(name)) return false; + while (end < text.length && this.isWhitespace(text[end])) end += 1; + return text[end] === '('; + } + + private isValidIdentifier(value: string) { + const tokens = this.tokensForCode(value); + + return ( + tokens.tokens.length === 1 && + tokens.tokenNames[0] === 'IDENTIFIER' && + tokens.tokens[0].text === value + ); + } + + private isIdentifierPart(value: string) { + return value.length > 0 && IDENTIFIER_PART_PATTERN.test(value); + } + + private isWhitespace(value: string | undefined) { + return value !== undefined && /\s/u.test(value); + } + + private codePointBefore(value: string, index: number) { + if (index === 0) return ''; + const previous = value.charCodeAt(index - 1); + const start = + previous >= 0xdc00 && previous <= 0xdfff ? index - 2 : index - 1; + return value.slice(start, index); + } + + private isEscapedCharacter(value: string, index: number) { + let backslashes = 0; + for ( + let position = index - 1; + position >= 0 && value[position] === '\\'; + position -= 1 + ) { + backslashes += 1; + } + + return backslashes % 2 === 1; + } + + private rangesOverlap(left: SourceRange, right: SourceRange) { + return left.start < right.end && right.start < left.end; + } + + private applyReplacements(code: string, replacements: SourceReplacement[]) { + return [...replacements] + .sort((left, right) => right.start - left.start) + .reduce( + (updated, replacement) => + updated.slice(0, replacement.start) + + replacement.text + + updated.slice(replacement.end), + code + ); + } +} diff --git a/TypeScript/src/__tests__/antlr4.interpreter.test.ts b/TypeScript/src/__tests__/antlr4.interpreter.test.ts index 8f55d96..f076d32 100644 --- a/TypeScript/src/__tests__/antlr4.interpreter.test.ts +++ b/TypeScript/src/__tests__/antlr4.interpreter.test.ts @@ -307,6 +307,32 @@ describe('Replace variables', () => { expect(result).toBe('
energy = global.stamina
'); }); + + test('rejects legacy characters instead of partially replacing identifiers', () => { + const interpreter = new Interpreter({ + state: { + accented: { + id: 'accented', + name: 'éscore', + type: 'integer', + defaultValue: 0, + }, + score: { + id: 'score', + name: 'score', + type: 'integer', + defaultValue: 0, + }, + }, + }); + + expect(() => + interpreter.replaceVariables( + '
éscore + score
', + { score: 'points' } + ) + ).toThrow(ParseError); + }); }); describe('Replace scopes', () => { diff --git a/TypeScript/src/__tests__/legacy-variable-reference-rewriter.test.ts b/TypeScript/src/__tests__/legacy-variable-reference-rewriter.test.ts new file mode 100644 index 0000000..01a6801 --- /dev/null +++ b/TypeScript/src/__tests__/legacy-variable-reference-rewriter.test.ts @@ -0,0 +1,624 @@ +import { describe, expect, test } from 'vitest'; +import { Interpreter } from '../index.js'; +import type { ArcscriptStateDef, VarDef } from '../types.js'; + +const variable = (id: string, name: string, scope?: string | null): VarDef => ({ + id, + name, + type: 'integer', + defaultValue: 0, + ...(scope === undefined ? {} : { scope }), +}); + +const rewrite = ( + state: ArcscriptStateDef, + code: string, + replacements: Record, + scopeNames?: readonly string[] +) => + new Interpreter({ state }).rewriteLegacyVariableReferences( + code, + replacements, + { scopeNames } + ); + +describe('legacy variable reference rewriting', () => { + test('rewrites invalid identifier shapes in raw Arcscript', () => { + const state = { + unicode: variable('unicode', 'Tariftyp_Gültig_ab'), + apostrophe: variable('apostrophe', "Amise'sPower"), + dollar: variable('dollar', '$ran$'), + dot: variable('dot', 'score.pl'), + whitespace: variable('whitespace', 'trucks destroyed'), + }; + + expect( + rewrite( + state, + "Tariftyp_Gültig_ab + Amise'sPower + $ran$ + score.pl + trucks destroyed", + { + unicode: 'Tariftyp_Gultig_ab', + apostrophe: 'Amise_sPower', + dollar: '$ran', + dot: 'score_pl', + whitespace: 'trucks_destroyed', + } + ) + ).toEqual({ + code: 'Tariftyp_Gultig_ab + Amise_sPower + $ran + score_pl + trucks_destroyed', + blockedVariableIds: [], + }); + }); + + test.each([ + ['participant-IL', 'participant_IL'], + ['Mean/Nice', 'Mean_Nice'], + ['pissTook?', 'pissTook'], + ['Brief Description:', 'Brief_Description'], + ['Sometimes connection is...', 'Sometimes_connection_is'], + ['Clue_Example_HugoHatesSpeakeasies.', 'Clue_Example_HugoHatesSpeakeasies'], + ['hospital.rooms.diagnosis', 'hospital_rooms_diagnosis'], + ['team_-1', 'team_1'], + ['max', 'max_variable'], + ['roll', 'roll_variable'], + ])('rewrites audited production name %s', (oldName, newName) => { + expect( + rewrite({ legacy: variable('legacy', oldName) }, `${oldName} == 1`, { + legacy: newName, + }) + ).toEqual({ + code: `${newName} == 1`, + blockedVariableIds: [], + }); + }); + + test('rewrites raw Arcscript without changing strings or mentions', () => { + const mention = + 'round'; + const code = `round < 3 and message == "score.pl" and visits(${mention})`; + + expect( + rewrite( + { + round: variable('round', 'round'), + score: variable('score', 'score.pl'), + }, + code, + { round: 'round_variable', score: 'score_pl' } + ) + ).toEqual({ + code: `round_variable < 3 and message == "score.pl" and visits(${mention})`, + blockedVariableIds: [], + }); + }); + + test('distinguishes reserved variable references from function calls', () => { + expect( + rewrite( + { round: variable('round', 'round') }, + 'round < 3 and round(2) == 2 and looked_around == true', + { round: 'round_variable' } + ) + ).toEqual({ + code: 'round_variable < 3 and round(2) == 2 and looked_around == true', + blockedVariableIds: [], + }); + }); + + test('uses lexer string and mention ranges without losing apostrophe names', () => { + const mention = + 'Bob\'s cat'; + + expect( + rewrite( + { + apostrophe: variable('apostrophe', "Amise'sPower"), + quoted: variable('quoted', 'score.pl'), + reserved: variable('reserved', 'round'), + }, + `Amise'sPower + Amise'sPower + "score.pl" + ${mention} + round`, + { + apostrophe: 'Amise_sPower', + quoted: 'score_pl', + reserved: 'round_variable', + } + ) + ).toEqual({ + code: `Amise_sPower + Amise_sPower + "score.pl" + ${mention} + round_variable`, + blockedVariableIds: [], + }); + }); + + test('preserves single-quoted literals after apostrophe names', () => { + const state = { + apostrophe: variable('apostrophe', "Amise'sPower"), + reserved: variable('reserved', 'round'), + }; + const replacements = { + apostrophe: 'Amise_sPower', + reserved: 'round_variable', + }; + + expect( + rewrite(state, "Amise'sPower + 'round' + round", replacements) + ).toEqual({ + code: "Amise_sPower + 'round' + round_variable", + blockedVariableIds: [], + }); + expect( + rewrite(state, `message == "Amise'sPower" and Amise'sPower`, replacements) + ).toEqual({ + code: `message == "Amise'sPower" and Amise_sPower`, + blockedVariableIds: [], + }); + }); + + test('preserves literals whose quotes are HTML encoded', () => { + const state = { reserved: variable('reserved', 'round') }; + const replacements = { reserved: 'round_variable' }; + + expect( + rewrite(state, 'message == "round" and round', replacements) + ).toEqual({ + code: 'message == "round" and round_variable', + blockedVariableIds: [], + }); + expect( + rewrite(state, 'message == 'round' and round', replacements) + ).toEqual({ + code: 'message == 'round' and round_variable', + blockedVariableIds: [], + }); + expect( + rewrite(state, 'show("hello "round"") and round', replacements) + ).toEqual({ + code: 'show("hello "round"") and round_variable', + blockedVariableIds: [], + }); + expect( + rewrite(state, "show('hello 'round'') and round", replacements) + ).toEqual({ + code: "show('hello 'round'') and round_variable", + blockedVariableIds: [], + }); + }); + + test('decodes only the entity beginning at the current position', () => { + expect( + rewrite({ unicode: variable('unicode', 'é') }, 'true && é > 1', { + unicode: 'e', + }) + ).toEqual({ + code: 'true && e > 1', + blockedVariableIds: [], + }); + }); + + test('does not let apostrophes in entity-delimited strings hide references', () => { + expect( + rewrite( + { + message: variable('message', 'message'), + unicode: variable('unicode', 'é'), + apostrophe: variable('apostrophe', "Amise'sPower"), + }, + "message == "Bob's cat" and é > 0 and Amise'sPower", + { unicode: 'e', apostrophe: 'Amise_sPower' } + ) + ).toEqual({ + code: "message == "Bob's cat" and e > 0 and Amise_sPower", + blockedVariableIds: [], + }); + }); + + test('preserves literal ranges after supplementary Unicode characters', () => { + expect( + rewrite({ unicode: variable('unicode', 'é') }, 'show("😀😀", "é", é)', { + unicode: 'e', + }) + ).toEqual({ + code: 'show("😀😀", "é", e)', + blockedVariableIds: [], + }); + }); + + test('preserves escaped quotes when masking legacy matches', () => { + expect( + rewrite( + { + escaped: variable('escaped', 'foo\\'), + reserved: variable('reserved', 'round'), + }, + 'show("foo\\"round") and round', + { escaped: 'foo', reserved: 'round_variable' } + ) + ).toEqual({ + code: 'show("foo\\"round") and round_variable', + blockedVariableIds: [], + }); + }); + + test('preserves complete escape sequences when masking legacy matches', () => { + expect( + rewrite({ unicode: variable('unicode', 'nöm') }, 'show("\\nöm", nöm)', { + unicode: 'nom', + }) + ).toEqual({ + code: 'show("\\nöm", nom)', + blockedVariableIds: [], + }); + }); + + test('does not create string delimiters while masking escaped quotes', () => { + expect( + rewrite( + { escaped: variable('escaped', 'foo\\"bar') }, + 'foo\\"bar + "foo\\"bar"', + { escaped: 'foo_bar' } + ) + ).toEqual({ + code: 'foo_bar + "foo\\"bar"', + blockedVariableIds: [], + }); + }); + + test('rewrites scoped references and preserves dot whitespace', () => { + const state = { + first: variable('first', 'score:name', 'one'), + second: variable('second', 'score:name', 'two'), + global: variable('global', 'score:name'), + }; + + expect( + rewrite( + state, + 'one.score:name + two . score:name + score:name', + { + first: 'score_name', + second: 'score_name_2', + global: 'score_name', + }, + ['one', 'two'] + ) + ).toEqual({ + code: 'one.score_name + two . score_name_2 + score_name', + blockedVariableIds: [], + }); + }); + + test('does not rewrite global names as member-expression parts', () => { + expect( + rewrite( + { round: variable('round', 'round') }, + 'round.score + unknownScope.round + round', + { round: 'round_variable' } + ) + ).toEqual({ + code: 'round.score + unknownScope.round + round_variable', + blockedVariableIds: [], + }); + }); + + test('blocks references that are valid Arcscript expressions', () => { + const state = { + score: variable('score', 'score'), + assignment: variable('assignment', 'score=1'), + words: variable('words', 'score is 1'), + numeric: variable('numeric', '123'), + expression: variable('expression', 'team_+1'), + team: variable('team', 'team_'), + }; + + expect( + rewrite(state, 'score=1; score is 1; 123 > 1; team_+1 == 2', { + assignment: 'score_1', + words: 'score_is_1', + numeric: '_123', + expression: 'team_1', + }) + ).toEqual({ + code: 'score=1; score is 1; 123 > 1; team_+1 == 2', + blockedVariableIds: ['assignment', 'expression', 'numeric', 'words'], + }); + }); + + test('blocks names that consume a surrounding syntax delimiter', () => { + expect( + rewrite( + { + score: variable('score', 'score'), + closing: variable('closing', 'score)'), + }, + 'show(score)', + { closing: 'score_value' } + ) + ).toEqual({ + code: 'show(score)', + blockedVariableIds: ['closing'], + }); + }); + + test('blocks names that consume an internal syntax delimiter', () => { + expect( + rewrite( + { + score: variable('score', 'score'), + bonus: variable('bonus', 'bonus'), + combined: variable('combined', 'score, bonus'), + }, + 'max(score, bonus)', + { combined: 'score_bonus' } + ) + ).toEqual({ + code: 'max(score, bonus)', + blockedVariableIds: ['combined'], + }); + }); + + test('blocks names containing an unescaped double quote', () => { + expect( + rewrite( + { + message: variable('message', 'message'), + legacy: variable('legacy', 'message = "hello'), + }, + 'message = "hello"', + { legacy: 'message_hello' } + ) + ).toEqual({ + code: 'message = "hello"', + blockedVariableIds: ['legacy'], + }); + }); + + test('blocks names containing a single-quoted string delimiter', () => { + expect( + rewrite( + { legacy: variable('legacy', "true is 'hello") }, + "true is 'hello'", + { legacy: 'true_is_hello' } + ) + ).toEqual({ + code: "true is 'hello'", + blockedVariableIds: ['legacy'], + }); + }); + + test('blocks padded names that resolve to existing syntax', () => { + expect( + rewrite( + { + score: variable('score', 'score'), + padded: variable('padded', 'score '), + boolean: variable('boolean', 'true '), + }, + 'score + 1; true + 1', + { padded: 'legacy_score', boolean: 'legacy_true' } + ) + ).toEqual({ + code: 'score + 1; true + 1', + blockedVariableIds: ['boolean', 'padded'], + }); + }); + + test('blocks scoped names that contain valid scoped expressions', () => { + expect( + rewrite( + { + score: variable('score', 'score', 'hero'), + assignment: variable('assignment', 'score=1', 'hero'), + team: variable('team', 'team_', 'hero'), + expression: variable('expression', 'team_+1', 'hero'), + }, + 'hero.score=1; hero.team_+1', + { assignment: 'score_1', expression: 'team_1' }, + ['hero'] + ) + ).toEqual({ + code: 'hero.score=1; hero.team_+1', + blockedVariableIds: ['assignment', 'expression'], + }); + }); + + test('blocks a pending legacy name that is also a valid if clause', () => { + expect( + rewrite( + { + condition: variable('condition', 'has_key'), + legacy: variable('legacy', 'if has_key'), + }, + 'if has_key', + { legacy: 'if_has_key' } + ) + ).toEqual({ + code: 'if has_key', + blockedVariableIds: ['legacy'], + }); + }); + + test('does not rewrite a whitespace name inside a longer phrase', () => { + expect( + rewrite( + { legacy: variable('legacy', 'Veteran Merc') }, + 'Veteran Merc Background', + { legacy: 'Veteran_Merc' } + ) + ).toEqual({ + code: 'Veteran Merc Background', + blockedVariableIds: [], + }); + }); + + test('does not rewrite a scoped whitespace name inside a longer phrase', () => { + expect( + rewrite( + { legacy: variable('legacy', 'Veteran Merc', 'hero') }, + 'hero.Veteran Merc Background', + { legacy: 'Veteran_Merc' }, + ['hero'] + ) + ).toEqual({ + code: 'hero.Veteran Merc Background', + blockedVariableIds: [], + }); + }); + + test('rewrites whitespace names next to Arcscript word operators', () => { + expect( + rewrite( + { + legacy: variable('legacy', 'trucks destroyed'), + other: variable('other', 'other'), + }, + 'if trucks destroyed; not trucks destroyed; other and trucks destroyed; trucks destroyed or other', + { legacy: 'trucks_destroyed' } + ) + ).toEqual({ + code: 'if trucks_destroyed; not trucks_destroyed; other and trucks_destroyed; trucks_destroyed or other', + blockedVariableIds: [], + }); + }); + + test('allows operator characters when they do not form a known expression', () => { + expect( + rewrite( + { + operator: variable('operator', 'team_+1'), + ampersand: variable('ampersand', 'A&B'), + }, + 'team_+1 == 2 and A&B == 1', + { operator: 'team_1', ampersand: 'A_B' } + ) + ).toEqual({ + code: 'team_1 == 2 and A_B == 1', + blockedVariableIds: [], + }); + }); + + test('blocks duplicate names and ambiguous scoped variables', () => { + expect( + rewrite( + { + first: variable('first', 'score name'), + second: variable('second', 'score name'), + scoped: variable('scoped', 'health value', 'duplicate'), + }, + 'score name + duplicate.health value', + { + first: 'score_name', + second: 'score_name_2', + scoped: 'health_value', + }, + ['duplicate', 'duplicate'] + ) + ).toEqual({ + code: 'score name + duplicate.health value', + blockedVariableIds: ['first', 'scoped', 'second'], + }); + }); + + test('blocks invalid and reserved scope names', () => { + expect( + rewrite( + { + operator: variable('operator', 'score name', 'foo-bar'), + reserved: variable('reserved', 'health value', 'round'), + }, + 'foo-bar.score name + round.health value', + { operator: 'score_name', reserved: 'health_value' }, + ['foo-bar', 'round'] + ) + ).toEqual({ + code: 'foo-bar.score name + round.health value', + blockedVariableIds: ['operator', 'reserved'], + }); + }); + + test('applies overlapping renames simultaneously without cascading', () => { + expect( + rewrite( + { + short: variable('short', 'a.b'), + long: variable('long', 'a.b.c'), + unicode: variable('unicode', 'ööö'), + generated: variable('generated', 'ooo+1'), + }, + 'a.b.c + a.b + ööö+1', + { + short: 'a_b', + long: 'a_b_c', + unicode: 'ooo', + generated: 'ooo_1', + } + ) + ).toEqual({ + code: 'a_b_c + a_b + ooo+1', + blockedVariableIds: [], + }); + }); + + test('does not match Unicode names inside longer Unicode names', () => { + expect( + rewrite( + { + first: variable('first', 'é'), + second: variable('second', 'àé'), + third: variable('third', 'éà'), + }, + 'àé + éà + é', + { first: 'e', second: 'ae', third: 'ea' } + ) + ).toEqual({ + code: 'ae + ea + e', + blockedVariableIds: [], + }); + }); + + test('blocks compound references to other legacy variables', () => { + expect( + rewrite( + { + component: variable('component', 'föo'), + compound: variable('compound', 'föo+1'), + }, + 'föo+1', + { component: 'foo', compound: 'foo_1' } + ) + ).toEqual({ + code: 'föo+1', + blockedVariableIds: ['compound'], + }); + expect( + rewrite( + { + component: variable('component', 'föo'), + compound: variable('compound', 'föo+1'), + }, + 'föo+1', + { compound: 'foo_1' } + ) + ).toEqual({ + code: 'föo+1', + blockedVariableIds: ['compound'], + }); + }); + + test('allows unreferenced legacy names but blocks invalid replacements', () => { + expect( + rewrite( + { + unreferenced: variable('unreferenced', 'missing name'), + invalidReplacement: variable('invalidReplacement', 'present name'), + }, + 'present name', + { + unreferenced: 'missing_name', + invalidReplacement: 'still invalid', + } + ) + ).toEqual({ + code: 'present name', + blockedVariableIds: ['invalidReplacement'], + }); + }); +}); diff --git a/TypeScript/src/antlr4.interpreter.ts b/TypeScript/src/antlr4.interpreter.ts index df30f16..7722f60 100644 --- a/TypeScript/src/antlr4.interpreter.ts +++ b/TypeScript/src/antlr4.interpreter.ts @@ -4,9 +4,15 @@ import ArcscriptLexer from './Generated/ArcscriptLexer.js'; import ArcscriptParser from './Generated/ArcscriptParser.js'; import ArcscriptVisitor from './ArcscriptVisitor.js'; import ErrorListener from './ErrorListener.js'; +import { ParseError } from './errors/index.js'; import { ArcscriptStateDef, VarValue } from './types.js'; import ArcscriptState from './ArcscriptState.js'; import { isGlobalScope } from './scope.js'; +import LegacyVariableReferenceRewriter from './LegacyVariableReferenceRewriter.js'; +import type { + LegacyVariableRewriteOptions, + LegacyVariableRewriteResult, +} from './types.js'; type ArcscriptInterpreterOptions = { state: ArcscriptStateDef; @@ -129,6 +135,22 @@ export default class Interpreter { return this.applyReplacements(code, replacements); } + /** + * Rewrites references to legacy variable names that the Arcscript grammar + * cannot parse as identifiers. Unlike replaceVariables(), unsafe ambiguous + * references are reported instead of being changed. + */ + rewriteLegacyVariableReferences( + code: string, + variables: Record, + options: LegacyVariableRewriteOptions = {} + ): LegacyVariableRewriteResult { + return new LegacyVariableReferenceRewriter( + this.arcscriptVariables, + legacyName => this.parsesAsArcscript(legacyName) + ).rewrite(code, variables, options); + } + replaceScope(code: string, scope: string, replacement: string) { return this.replaceScopes(code, { [scope]: replacement, @@ -158,7 +180,8 @@ export default class Interpreter { return stateVars.some( variable => - variable.scope === token.text && variable.name === variableToken.text + variable.scope === token.text && + variable.name === variableToken.text ); }) .filter(scopeToken => @@ -187,17 +210,47 @@ export default class Interpreter { private parseTokens(code: string) { const { lexer } = this.createLexer(code); + const tokenTypeNames = lexer.getSymbolicNames(); + const allTokens = lexer.getAllTokens(); + const legacyToken = allTokens.find( + token => tokenTypeNames[token.type] === 'LEGACY_CHARACTER' + ); + if (legacyToken) { + throw new ParseError(`token recognition error at: '${legacyToken.text}'`, { + line: legacyToken.line, + col: legacyToken.column, + }); + } return { - tokenTypeNames: lexer.getSymbolicNames(), - allTokens: lexer.getAllTokens(), + tokenTypeNames, + allTokens, }; } - private applyReplacements( - code: string, - replacements: SourceReplacement[] - ) { + private parsesAsArcscript(code: string) { + const scripts = [ + `
${code}
`, + `
${code}

endif
`, + `
if true

${code}

endif
`, + ]; + + for (const script of scripts) { + try { + this.parse(script); + + return true; + } catch (error) { + if (!(error instanceof ParseError)) { + throw error; + } + } + } + + return false; + } + + private applyReplacements(code: string, replacements: SourceReplacement[]) { return [...replacements] .sort((a, b) => b.start - a.start) .reduce( diff --git a/TypeScript/src/index.ts b/TypeScript/src/index.ts index e9032ac..a2416cd 100644 --- a/TypeScript/src/index.ts +++ b/TypeScript/src/index.ts @@ -1,4 +1,9 @@ import Interpreter from './antlr4.interpreter.js'; import { RuntimeError, ParseError, FatalError } from './errors/index.js'; +import type { + LegacyVariableRewriteOptions, + LegacyVariableRewriteResult, +} from './types.js'; export { Interpreter, RuntimeError, ParseError, FatalError }; +export type { LegacyVariableRewriteOptions, LegacyVariableRewriteResult }; diff --git a/TypeScript/src/types.ts b/TypeScript/src/types.ts index f559a3d..0b6079f 100644 --- a/TypeScript/src/types.ts +++ b/TypeScript/src/types.ts @@ -16,3 +16,16 @@ export type MentionResult = { }; export type ArcscriptStateDef = Record; + +export type LegacyVariableRewriteOptions = { + /** + * All board/component scope names in the project. Keep duplicates so an + * ambiguous scope shared by more than one container can be detected. + */ + scopeNames?: readonly string[]; +}; + +export type LegacyVariableRewriteResult = { + code: string; + blockedVariableIds: string[]; +}; diff --git a/generate.ps1 b/generate.ps1 index 041e71d..6fcb707 100644 --- a/generate.ps1 +++ b/generate.ps1 @@ -9,6 +9,9 @@ Set-Location "grammar" java -Xmx500M -cp ../antlr4.jar org.antlr.v4.Tool -Dlanguage=TypeScript ArcscriptLexer.g4 ArcscriptParser.g4 -visitor -no-listener -o ./TypeScript New-Item -ItemType Directory -Force -Path ../TypeScript/src/Generated | Out-Null Copy-Item TypeScript\*.ts ../TypeScript/src/Generated\ +(Get-Content ../TypeScript/src/Generated/ArcscriptParser.ts) ` + -replace "'./ArcscriptParserBase'", "'./ArcscriptParserBase.js'" | ` + Set-Content ../TypeScript/src/Generated/ArcscriptParser.ts Remove-Item -Recurse -Force ./TypeScript # Copy grammar files for Cpp @@ -30,4 +33,4 @@ Set-Location .. java -Xmx500M -cp ../antlr4.jar org.antlr.v4.Tool -Dlanguage=CSharp ArcscriptLexer.g4 ArcscriptParser.g4 -visitor -no-listener -o ./CSharp -package Arcweave.Interpreter New-Item -ItemType Directory -Force -Path ../CSharp/Interpreter/Generated | Out-Null Copy-Item CSharp\*.cs ../CSharp/Interpreter/Generated\ -Remove-Item -Recurse -Force ./CSharp \ No newline at end of file +Remove-Item -Recurse -Force ./CSharp diff --git a/generate.sh b/generate.sh index 944e4d1..c874b84 100644 --- a/generate.sh +++ b/generate.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash +set -e + cd "$(dirname "$0")" if [ ! -f ./antlr4.jar ] @@ -9,6 +11,9 @@ cd grammar java -Xmx500M -cp ../antlr4.jar org.antlr.v4.Tool -Dlanguage=TypeScript ArcscriptLexer.g4 ArcscriptParser.g4 -visitor -no-listener -o ./TypeScript mkdir -p ../TypeScript/src/Generated cp TypeScript/*.ts ../TypeScript/src/Generated/. +parser_file=../TypeScript/src/Generated/ArcscriptParser.ts +sed "s|'./ArcscriptParserBase'|'./ArcscriptParserBase.js'|" "$parser_file" > "$parser_file.tmp" +mv "$parser_file.tmp" "$parser_file" rm -rf ./TypeScript cp Arcscript*.g4 ./Cpp/. @@ -28,4 +33,4 @@ cd .. java -Xmx500M -cp ../antlr4.jar org.antlr.v4.Tool -Dlanguage=CSharp ArcscriptLexer.g4 ArcscriptParser.g4 -visitor -no-listener -o ./CSharp -package Arcweave.Interpreter mkdir -p ../CSharp/Interpreter/Generated cp CSharp/*.cs ../CSharp/Interpreter/Generated/. -rm -rf ./CSharp \ No newline at end of file +rm -rf ./CSharp diff --git a/grammar/ArcscriptLexer.g4 b/grammar/ArcscriptLexer.g4 index 86eb754..63cc408 100644 --- a/grammar/ArcscriptLexer.g4 +++ b/grammar/ArcscriptLexer.g4 @@ -100,8 +100,11 @@ ISKEYWORD: 'is'; NOTKEYWORD: 'not'; -STRING: '"' STRING_CONTENT* '"' | '\'' STRING_CONTENT* '\''; -fragment STRING_CONTENT: ~[\\\r\n'"] | '\\' [abfnrtv'"\\]; +STRING: + '"' DOUBLE_STRING_CONTENT* '"' + | '\'' SINGLE_STRING_CONTENT* '\''; +fragment DOUBLE_STRING_CONTENT: ~[\\\r\n"] | '\\' [abfnrtv'"\\]; +fragment SINGLE_STRING_CONTENT: ~[\\\r\n'] | '\\' [abfnrtv'"\\]; IDENTIFIER: IDENTIFIER_START IDENTIFIER_PART*; @@ -110,6 +113,10 @@ fragment IDENTIFIER_PART: [0-9A-Za-z$_]; WHITESPACE: [ \t\r\n]+ -> skip; +// Keep tokenizing malformed legacy code so source-rewrite tooling can inspect +// the complete code block. The parser still rejects this token. +LEGACY_CHARACTER: .; + mode MENTIONSEGMENT; TAG_CLOSE: '>' -> pushMode(MENTIONLABELSEGMENT); @@ -161,4 +168,4 @@ fragment DECCHARS: [0-9]+ '%'?; fragment DOUBLE_QUOTE_STRING: '"' ~[<"]* '"'; -fragment SINGLE_QUOTE_STRING: '\'' ~[<']* '\''; \ No newline at end of file +fragment SINGLE_QUOTE_STRING: '\'' ~[<']* '\'';