Remove iconv and patch tests

This commit is contained in:
Anton Liaposhchenko 2024-12-29 00:06:24 +02:00
parent 4c0fbd9572
commit 8b23f3c82d
12 changed files with 29 additions and 91 deletions
src
main/lib
extension/core
helpers
markup.ts
test/tests
integration
comparison
filters
convert_encoding
index.ts
functions/random
unit/lib/helpers/iconv

@ -2,9 +2,12 @@ import {iconv} from "../../../helpers/iconv";
import {TwingSynchronousCallable} from "../../../callable-wrapper";
export const convertEncodingSynchronously: TwingSynchronousCallable<[
value: string | Buffer,
value: string | BufferSource,
to: string,
from: string
], Buffer> = (_executionContext, value, to, from) => {
return iconv(from, to, Buffer.from(value));
], Uint8Array> = (_executionContext, value, to, from) => {
if (typeof value === 'string') {
value = new TextEncoder().encode(value)
}
return iconv(from, to, value as BufferSource);
};

@ -1,4 +1,3 @@
import {iconv} from "../../../helpers/iconv";
import {isTraversable} from "../../../helpers/is-traversable";
import {iteratorToArray} from "../../../helpers/iterator-to-array";
import {TwingSynchronousCallable} from "../../../callable-wrapper";
@ -21,10 +20,7 @@ const array_rand = require('locutus/php/array/array_rand');
*
* @returns {Promise<any>} A random value from the given sequence
*/
export const randomSynchronously: TwingSynchronousCallable = (executionContext, values: any | null, max: number | null): any => {
const {environment} = executionContext;
const {charset} = environment;
export const randomSynchronously: TwingSynchronousCallable = (_executionContext, values: any | null, max: number | null): any => {
if (values === null) {
return max === null ? mt_rand() : mt_rand(0, max);
}
@ -39,26 +35,16 @@ export const randomSynchronously: TwingSynchronousCallable = (executionContext,
}
if (typeof values === 'string') {
values = Buffer.from(values);
values = new TextEncoder().encode(values);
}
if (Buffer.isBuffer(values)) {
if (values.toString() === '') {
if (ArrayBuffer.isView(values)) {
if (values.byteLength === 0) {
return '';
}
if (charset !== 'UTF-8') {
values = iconv(charset, 'UTF-8', values);
}
// unicode split
values = runes(values.toString());
if (charset !== 'UTF-8') {
values = values.map((value: string) => {
return iconv('UTF-8', charset, Buffer.from(value)).toString();
});
}
values = runes(new TextDecoder().decode(values));
}
else if (isTraversable(values)) {
values = iteratorToArray(values);

@ -10,7 +10,7 @@ import {isAMapLike, MapLike} from "./map-like";
import {isAMarkup, TwingMarkup} from "../markup";
import {iteratorToMap} from "./iterator-to-map";
type Operand = Buffer | TwingMarkup | DateTime | MapLike<any, any> | string | boolean | number | null | object | Array<any>;
type Operand = ArrayBufferView | TwingMarkup | DateTime | MapLike<any, any> | string | boolean | number | null | object | Array<any>;
export function compare(
firstOperand: Operand,
@ -62,12 +62,12 @@ export function compare(
}
// Buffer
if (Buffer.isBuffer(firstOperand)) {
firstOperand = firstOperand.toString();
if (ArrayBuffer.isView(firstOperand)) {
firstOperand = new TextDecoder().decode(firstOperand);
}
if (Buffer.isBuffer(secondOperand)) {
secondOperand = secondOperand.toString();
if (ArrayBuffer.isView(secondOperand)) {
secondOperand = new TextDecoder().decode(secondOperand);
}
// Map

@ -1,5 +1,3 @@
const {decode, encode} = require('iconv-lite');
/**
* Internationalization conversion: convert buffer to requested character encoding
*
@ -9,10 +7,14 @@ const {decode, encode} = require('iconv-lite');
*
* @returns {Buffer} the converted buffer or false on failure.
*/
export function iconv(inCharset: string, outCharset: string, buffer: Buffer): Buffer {
let str = decode(buffer, inCharset);
export function iconv(inCharset: string, outCharset: string, buffer: BufferSource): Uint8Array {
if (outCharset !== 'UTF-8') {
throw new Error('Only UTF-8 is supported as the output charset');
}
buffer = encode(str, outCharset) as Buffer;
const decoder = new TextDecoder(inCharset.toLowerCase())
const str = decoder.decode(buffer);
return buffer;
const encoder = new TextEncoder();
return encoder.encode(str);
}

@ -1,5 +1,3 @@
import {iconv} from "./helpers/iconv";
export interface TwingMarkup {
readonly charset: string;
readonly content: string;
@ -31,9 +29,7 @@ export const createMarkup = (
return charset;
},
get count() {
const contentAsString = iconv(charset, 'utf-8', Buffer.from(content)).toString();
return contentAsString.length;
return content.length;
},
toString() {
return content.toString();

@ -4,7 +4,7 @@ runTest({
description: 'Comparison of something to a buffer',
templates: {
"index.twig": `
{{ "foo"|convert_encoding("ISO-8859-1", "UTF-8") == "foo" }}
{{ "foo"|convert_encoding("UTF-8", "ISO-8859-1") == "foo" }}
`
},
trimmedExpectation: `

@ -22,7 +22,7 @@ runTest({
{{ "f" == ["f"] ? 1 : 0 }}
{{ "f" == ["foo"] ? 1 : 0 }}
{{ "foo" == "foo"|convert_encoding("ISO-8859-1", "UTF-8") }}
{{ "foo" == "foo"|convert_encoding("UTF-8", "ISO-8859-1") }}
{{ "foo" == "foo"|escape }}
`
},

@ -1,27 +0,0 @@
import TestBase, {runTest} from "../../TestBase";
import {createIntegrationTest} from "../../test";
class Test extends TestBase {
getDescription() {
return '"convert_encoding" filter';
}
getTemplates() {
return {
'index.twig': `{#
we won't support ISO-2022-JP since it implies using iconv
ISO-2022-JP support will eventually be part of an extension
#}
{#{{ "愛していますか?"|convert_encoding('ISO-2022-JP', 'UTF-8')|convert_encoding('UTF-8', 'ISO-2022-JP') }}#}
{{ "愛していますか?"|convert_encoding('Shift_JIS', 'UTF-8')|convert_encoding('UTF-8', 'Shift_JIS') }}`
};
}
getExpected() {
return `
`;
}
}
runTest(createIntegrationTest(new Test()));

@ -2,7 +2,6 @@ import "./abs";
import "./batch";
import "./capitalize";
import "./column";
import "./convert_encoding";
import "./date";
import "./date_modify";
import "./default";

@ -1,5 +1,4 @@
import {runTest} from "../../TestBase";
import {iconv} from "../../../../../main/lib/helpers/iconv";
runTest({
description: '"random" function with buffer as values',
@ -19,22 +18,3 @@ runTest({
1
`
});
runTest({
description: '"random" function with buffer as values against a non UTF-8 runtime',
templates: {
"index.twig": `
{% set random1 = random(buffer) %}
{{ random1 == "Ä"|convert_encoding('ISO-8859-1', 'UTF-8') or random1 == "é"|convert_encoding('ISO-8859-1', 'UTF-8') }}
`
},
synchronousContext: {
buffer: iconv('UTF-8', 'ISO-8859-1', Buffer.from('Äé')),
},
synchronousEnvironmentOptions: {
charset: 'ISO-8859-1'
},
trimmedExpectation: `
1
`
});

@ -2,8 +2,8 @@ import tape from 'tape';
import {iconv} from "../../../../../../main/lib/helpers/iconv";
tape('iconv', ({test}) => {
test('is non destructive', ({same, end}) => {
same(iconv('ISO-8859-1', 'UTF-8', iconv('UTF-8', 'ISO-8859-1', Buffer.from('Äé'))).toString(), 'Äé');
test('converts from ISO-8859-1 to UTF-8', ({same, end}) => {
same(iconv('ISO-8859-1', 'UTF-8', new Uint8Array([0xC4])), new Uint8Array([0xC3, 0x84]))
end();
});