diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3b3fc82..4de6b26 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -293,13 +293,14 @@ jobs: working-directory: examples/with-javascript-browser run: bun i && bun test.cjs - - name: deno with-javascript-browser - if: matrix.os != 'windows-latest' && matrix.os != 'LinuxARM64' #https://github.com/denoland/deno/issues/23524#issuecomment-2292075726 - uses: nick-fields/retry@v3 #doing this step with the retry action because sometimes in macos it gets stuck without failing - with: - timeout_seconds: 45 - max_attempts: 5 - command: cd examples/with-javascript-browser && deno --allow-all test.cjs + # See issue https://github.com/sqlitecloud/sqlitecloud-js/issues/265 + # - name: deno with-javascript-browser + # if: matrix.os != 'windows-latest' && matrix.os != 'LinuxARM64' #https://github.com/denoland/deno/issues/23524#issuecomment-2292075726 + # uses: nick-fields/retry@v3 #doing this step with the retry action because sometimes in macos it gets stuck without failing + # with: + # timeout_seconds: 45 + # max_attempts: 5 + # command: cd examples/with-javascript-browser && deno --allow-all test.cjs - name: remove with-javascript-browser run: rm -rf examples/with-javascript-browser/* diff --git a/package-lock.json b/package-lock.json index 3f5ac5d..c6960f5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@sqlitecloud/drivers", - "version": "1.0.739", + "version": "1.0.834", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@sqlitecloud/drivers", - "version": "1.0.739", + "version": "1.0.834", "license": "MIT", "dependencies": { "buffer": "^6.0.3", @@ -17326,4 +17326,4 @@ } } } -} \ No newline at end of file +} diff --git a/package.json b/package.json index e1d86a5..0673e19 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@sqlitecloud/drivers", - "version": "1.0.779", + "version": "1.0.834", "description": "SQLiteCloud drivers for Typescript/Javascript in edge, web and node clients", "main": "./lib/index.js", "types": "./lib/index.d.ts", diff --git a/src/drivers/connection-tls.ts b/src/drivers/connection-tls.ts index b6f2119..6c2d798 100644 --- a/src/drivers/connection-tls.ts +++ b/src/drivers/connection-tls.ts @@ -221,17 +221,17 @@ export class SQLiteCloudTlsConnection extends SQLiteCloudConnection { this.processCommandsData(Buffer.alloc(0)) return } else { - const { data } = popData(decompressResults.buffer) + const { data } = popData(decompressResults.buffer, this.config.safe_integer_mode) this.processCommandsFinish?.call(this, null, data) } } else { if (dataType !== CMD_ROWSET_CHUNK) { - const { data } = popData(this.buffer) + const { data } = popData(this.buffer, this.config.safe_integer_mode) this.processCommandsFinish?.call(this, null, data) } else { const completeChunk = bufferEndsWith(this.buffer, ROWSET_CHUNKS_END) if (completeChunk) { - const parsedData = parseRowsetChunks([...this.pendingChunks, this.buffer]) + const parsedData = parseRowsetChunks([...this.pendingChunks, this.buffer], this.config.safe_integer_mode) this.processCommandsFinish?.call(this, null, parsedData) } } @@ -241,7 +241,7 @@ export class SQLiteCloudTlsConnection extends SQLiteCloudConnection { // command with no explicit len so make sure that the final character is a space const lastChar = this.buffer.subarray(this.buffer.length - 1, this.buffer.length).toString('utf8') if (lastChar == ' ') { - const { data } = popData(this.buffer) + const { data } = popData(this.buffer, this.config.safe_integer_mode) this.processCommandsFinish?.call(this, null, data) } } diff --git a/src/drivers/protocol.ts b/src/drivers/protocol.ts index 2486c43..13d0926 100644 --- a/src/drivers/protocol.ts +++ b/src/drivers/protocol.ts @@ -3,7 +3,14 @@ // import { SQLiteCloudRowset } from './rowset' -import { SAFE_INTEGER_MODE, SQLiteCloudCommand, SQLiteCloudError, type SQLCloudRowsetMetadata, type SQLiteCloudDataTypes } from './types' +import { + SAFE_INTEGER_MODE, + SQLiteCloudCommand, + SQLiteCloudError, + type SQLCloudRowsetMetadata, + type SQLiteCloudDataTypes, + type SQLiteCloudSafeIntegerMode +} from './types' import { getSafeBuffer } from './safe-imports' // explicitly importing buffer library to allow cross-platform support by replacing it @@ -125,7 +132,7 @@ export function parseError(buffer: Buffer, spaceIndex: number): never { } /** Parse an array of items (each of which will be parsed by type separately) */ -export function parseArray(buffer: Buffer, spaceIndex: number): SQLiteCloudDataTypes[] { +export function parseArray(buffer: Buffer, spaceIndex: number, safeIntegerMode: SQLiteCloudSafeIntegerMode = SAFE_INTEGER_MODE): SQLiteCloudDataTypes[] { const parsedData = [] const array = buffer.subarray(spaceIndex + 1, buffer.length) @@ -133,7 +140,7 @@ export function parseArray(buffer: Buffer, spaceIndex: number): SQLiteCloudDataT let arrayItems = array.subarray(array.indexOf(' ') + 1, array.length) for (let i = 0; i < numberOfItems; i++) { - const { data, fwdBuffer: buffer } = popData(arrayItems) + const { data, fwdBuffer: buffer } = popData(arrayItems, safeIntegerMode) parsedData.push(data) arrayItems = buffer } @@ -165,9 +172,9 @@ export function parseRowsetHeader(buffer: Buffer): { index: number; metadata: SQ } /** Extract column names and, optionally, more metadata out of a rowset's header */ -function parseRowsetColumnsMetadata(buffer: Buffer, metadata: SQLCloudRowsetMetadata): Buffer { +function parseRowsetColumnsMetadata(buffer: Buffer, metadata: SQLCloudRowsetMetadata, safeIntegerMode: SQLiteCloudSafeIntegerMode): Buffer { function popForward() { - const { data, fwdBuffer: fwdBuffer } = popData(buffer) // buffer in parent scope + const { data, fwdBuffer: fwdBuffer } = popData(buffer, safeIntegerMode) // buffer in parent scope buffer = fwdBuffer return data } @@ -192,16 +199,16 @@ function parseRowsetColumnsMetadata(buffer: Buffer, metadata: SQLCloudRowsetMeta } /** Parse a regular rowset (no chunks) */ -function parseRowset(buffer: Buffer, spaceIndex: number): SQLiteCloudRowset { +function parseRowset(buffer: Buffer, spaceIndex: number, safeIntegerMode: SQLiteCloudSafeIntegerMode): SQLiteCloudRowset { buffer = buffer.subarray(spaceIndex + 1, buffer.length) const { metadata, fwdBuffer } = parseRowsetHeader(buffer) - buffer = parseRowsetColumnsMetadata(fwdBuffer, metadata) + buffer = parseRowsetColumnsMetadata(fwdBuffer, metadata, safeIntegerMode) // decode each rowset item const data = [] for (let j = 0; j < metadata.numberOfRows * metadata.numberOfColumns; j++) { - const { data: rowData, fwdBuffer } = popData(buffer) + const { data: rowData, fwdBuffer } = popData(buffer, safeIntegerMode) data.push(rowData) buffer = fwdBuffer } @@ -223,7 +230,7 @@ export function bufferEndsWith(buffer: Buffer, suffix: string): boolean { * *LEN 0:VERS NROWS NCOLS DATA * @see https://github.com/sqlitecloud/sdk/blob/master/PROTOCOL.md#scsp-rowset-chunk */ -export function parseRowsetChunks(buffers: Buffer[]): SQLiteCloudRowset { +export function parseRowsetChunks(buffers: Buffer[], safeIntegerMode: SQLiteCloudSafeIntegerMode = SAFE_INTEGER_MODE): SQLiteCloudRowset { let buffer = Buffer.concat(buffers) if (!bufferStartsWith(buffer, CMD_ROWSET_CHUNK) || !bufferEndsWith(buffer, ROWSET_CHUNKS_END)) { throw new Error('SQLiteCloudConnection.parseRowsetChunks - invalid chunks buffer') @@ -245,14 +252,14 @@ export function parseRowsetChunks(buffers: Buffer[]): SQLiteCloudRowset { // first chunk? extract columns metadata if (chunkIndex === 1) { metadata = chunkMetadata - buffer = parseRowsetColumnsMetadata(buffer, metadata) + buffer = parseRowsetColumnsMetadata(buffer, metadata, safeIntegerMode) } else { metadata.numberOfRows += chunkMetadata.numberOfRows } // extract single rowset row for (let k = 0; k < chunkMetadata.numberOfRows * metadata.numberOfColumns; k++) { - const { data: itemData, fwdBuffer } = popData(buffer) + const { data: itemData, fwdBuffer } = popData(buffer, safeIntegerMode) data.push(itemData) buffer = fwdBuffer } @@ -276,7 +283,10 @@ function popIntegers(buffer: Buffer, numberOfIntegers = 1): { data: number[]; fw } /** Parse command, extract its data, return the data and the buffer moved to the first byte after the command */ -export function popData(buffer: Buffer): { data: SQLiteCloudDataTypes | SQLiteCloudRowset; fwdBuffer: Buffer } { +export function popData( + buffer: Buffer, + safeIntegerMode: SQLiteCloudSafeIntegerMode = SAFE_INTEGER_MODE +): { data: SQLiteCloudDataTypes | SQLiteCloudRowset; fwdBuffer: Buffer } { function popResults(data: any) { const fwdBuffer = buffer.subarray(commandEnd) return { data, fwdBuffer } @@ -307,10 +317,10 @@ export function popData(buffer: Buffer): { data: SQLiteCloudDataTypes | SQLiteCl case CMD_INT: // SQLite uses 64-bit INTEGER, but JS uses 53-bit Number const value = BigInt(buffer.subarray(1, spaceIndex).toString()) - if (SAFE_INTEGER_MODE === 'bigint') { + if (safeIntegerMode === 'bigint') { return popResults(value) } - if (SAFE_INTEGER_MODE === 'mixed') { + if (safeIntegerMode === 'mixed') { if (value <= BigInt(Number.MIN_SAFE_INTEGER) || BigInt(Number.MAX_SAFE_INTEGER) <= value) { return popResults(value) } @@ -333,9 +343,9 @@ export function popData(buffer: Buffer): { data: SQLiteCloudDataTypes | SQLiteCl case CMD_BLOB: return popResults(buffer.subarray(spaceIndex + 1, commandEnd)) case CMD_ARRAY: - return popResults(parseArray(buffer, spaceIndex)) + return popResults(parseArray(buffer, spaceIndex, safeIntegerMode)) case CMD_ROWSET: - return popResults(parseRowset(buffer, spaceIndex)) + return popResults(parseRowset(buffer, spaceIndex, safeIntegerMode)) case CMD_ERROR: parseError(buffer, spaceIndex) // throws custom error break diff --git a/src/drivers/types.ts b/src/drivers/types.ts index 3e5e4d9..f251a15 100644 --- a/src/drivers/types.ts +++ b/src/drivers/types.ts @@ -19,9 +19,14 @@ export const DEFAULT_PORT = 8860 * (inlcuding `lastID` from WRITE statements) * mixed - use BigInt and Number types depending on the value size */ -export let SAFE_INTEGER_MODE = 'number' +export type SQLiteCloudSafeIntegerMode = 'number' | 'bigint' | 'mixed' + +export let SAFE_INTEGER_MODE: SQLiteCloudSafeIntegerMode = 'number' if (typeof process !== 'undefined') { - SAFE_INTEGER_MODE = process.env['SAFE_INTEGER_MODE']?.toLowerCase() || 'number' + const mode = process.env['SAFE_INTEGER_MODE']?.toLowerCase() + if (mode === 'bigint' || mode === 'mixed' || mode === 'number') { + SAFE_INTEGER_MODE = mode + } } if (SAFE_INTEGER_MODE == 'bigint') { console.debug('BigInt mode: Using Number for all INTEGER values from SQLite, including meta information from WRITE statements.') @@ -79,6 +84,8 @@ export interface SQLiteCloudConfig { maxrows?: number /** Server should limit total number of rows in a set to maxRowset */ maxrowset?: number + /** How SQLite 64-bit INTEGER values are returned: number, bigint or mixed. Defaults to SAFE_INTEGER_MODE env var, then number */ + safe_integer_mode?: SQLiteCloudSafeIntegerMode /** Custom options and configurations for tls socket, eg: additional certificates */ tlsoptions?: tls.ConnectionOptions diff --git a/src/drivers/utilities.ts b/src/drivers/utilities.ts index 225d84b..c56649f 100644 --- a/src/drivers/utilities.ts +++ b/src/drivers/utilities.ts @@ -2,7 +2,16 @@ // utilities.ts - utility methods to manipulate SQL statements // -import { DEFAULT_PORT, DEFAULT_TIMEOUT, SQLiteCloudArrayType, SQLiteCloudConfig, SQLiteCloudDataTypes, SQLiteCloudError } from './types' +import { + DEFAULT_PORT, + DEFAULT_TIMEOUT, + SAFE_INTEGER_MODE, + SQLiteCloudArrayType, + SQLiteCloudConfig, + SQLiteCloudDataTypes, + SQLiteCloudError, + SQLiteCloudSafeIntegerMode +} from './types' import { getSafeURL } from './safe-imports' // explicitly importing these libraries to allow cross-platform support by replacing them @@ -174,6 +183,7 @@ export function validateConfiguration(config: SQLiteCloudConfig): SQLiteCloudCon config.verbose = parseBoolean(config.verbose) config.noblob = parseBoolean(config.noblob) config.compression = config.compression != undefined && config.compression != null ? parseBoolean(config.compression) : true // default: true + config.safe_integer_mode = parseSafeIntegerMode(config.safe_integer_mode || SAFE_INTEGER_MODE) config.create = parseBoolean(config.create) config.non_linearizable = parseBoolean(config.non_linearizable) @@ -242,6 +252,7 @@ export function parseconnectionstring(connectionstring: string): SQLiteCloudConf maxdata: options.maxdata ? parseInt(options.maxdata) : undefined, maxrows: options.maxrows ? parseInt(options.maxrows) : undefined, maxrowset: options.maxrowset ? parseInt(options.maxrowset) : undefined, + safe_integer_mode: options.safe_integer_mode ? parseSafeIntegerMode(options.safe_integer_mode) : undefined, usewebsocket: options.usewebsocket ? parseBoolean(options.usewebsocket) : undefined, verbose: options.verbose ? parseBoolean(options.verbose) : undefined } @@ -278,3 +289,12 @@ export function parseBooleanToZeroOne(value: string | boolean | null | undefined } return value ? 1 : 0 } + +/** Parse 64-bit integer handling mode */ +export function parseSafeIntegerMode(value: string | SQLiteCloudSafeIntegerMode | null | undefined): SQLiteCloudSafeIntegerMode { + const mode = value?.toLowerCase() + if (mode === 'number' || mode === 'bigint' || mode === 'mixed') { + return mode + } + return 'number' +} diff --git a/src/index.ts b/src/index.ts index 3eb0db0..64de62a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -11,6 +11,7 @@ export { Database } from './drivers/database' export { SQLiteCloudConnection } from './drivers/connection' export { type SQLiteCloudConfig, + type SQLiteCloudSafeIntegerMode, type SQLCloudRowsetMetadata, SQLiteCloudError, type ResultsCallback, diff --git a/test/connection-tls.test.ts b/test/connection-tls.test.ts index c72cc34..85969b3 100644 --- a/test/connection-tls.test.ts +++ b/test/connection-tls.test.ts @@ -341,6 +341,33 @@ describe('send test commands', () => { }) }) + it('should use safe integer mode from connection string params', done => { + const config = getChinookConfig() + const connectionUrl = getChinookApiKeyUrl() + const separator = connectionUrl.includes('?') ? '&' : '?' + config.connectionstring = `${connectionUrl}${separator}safe_integer_mode=bigint` + + const chinook = new SQLiteCloudTlsConnection(config, error => { + if (error) { + done(error) + return + } + + chinook.sendCommands('TEST INTEGER', (error, results) => { + let err = null + try { + expect(error).toBeNull() + expect(results).toBe(BigInt(123456)) + } catch (error) { + err = error + } finally { + chinook.close() + err ? done(err) : done() + } + }) + }) + }) + it('should test null', done => { const connection = getConnection() connection.sendCommands('TEST NULL', (error, results) => { diff --git a/test/protocol.test.ts b/test/protocol.test.ts index 0e9f3fd..2ce2b57 100644 --- a/test/protocol.test.ts +++ b/test/protocol.test.ts @@ -2,7 +2,7 @@ // protocol.test.ts // -import { formatCommand, parseRowsetChunks } from '../src/drivers/protocol' +import { formatCommand, parseRowsetChunks, popData } from '../src/drivers/protocol' import { SQLiteCloudCommand } from '../src/drivers/types' // response sent by the server when we TEST ROWSET_CHUNK @@ -31,6 +31,30 @@ describe('parseRowsetChunks', () => { }) }) +describe('Safe integer mode', () => { + it('should return numbers by default', () => { + const { data } = popData(Buffer.from(':9007199254740992 ')) + expect(data).toBe(9007199254740992) + expect(typeof data).toBe('number') + }) + + it('should return bigint when mode is bigint', () => { + const { data } = popData(Buffer.from(':42 '), 'bigint') + expect(data).toBe(BigInt(42)) + expect(typeof data).toBe('bigint') + }) + + it('should return bigint only for unsafe integers when mode is mixed', () => { + const small = popData(Buffer.from(':42 '), 'mixed') + const large = popData(Buffer.from(':9007199254740992 '), 'mixed') + + expect(small.data).toBe(42) + expect(typeof small.data).toBe('number') + expect(large.data).toBe(BigInt('9007199254740992')) + expect(typeof large.data).toBe('bigint') + }) +}) + const testCases = [ { query: "SELECT 'hello world'", parameters: [], expected: "+20 SELECT 'hello world'" }, { diff --git a/test/utilities.test.ts b/test/utilities.test.ts index f3e26b5..0da8df8 100644 --- a/test/utilities.test.ts +++ b/test/utilities.test.ts @@ -3,7 +3,7 @@ // import { SQLiteCloudError } from '../src/index' -import { getInitializationCommands, parseconnectionstring, sanitizeSQLiteIdentifier } from '../src/drivers/utilities' +import { getInitializationCommands, parseconnectionstring, sanitizeSQLiteIdentifier, validateConfiguration } from '../src/drivers/utilities' import { getTestingDatabaseName } from './shared' import { expect, describe, it } from '@jest/globals' @@ -163,6 +163,13 @@ describe('parseconnectionstring', () => { expect(config.timeout).toBe(123) }) + it('should parse connection with safe integer mode', () => { + const connectionstring = `sqlitecloud://host:1234/database?apikey=xxx&safe_integer_mode=bigint` + const config = parseconnectionstring(connectionstring) + + expect(config.safe_integer_mode).toBe('bigint') + }) + it('expect error when both user/pass and api key are set', () => { const connectionstring = 'sqlitecloud://user:password@host:1234/database?apikey=yyy' expect(() => parseconnectionstring(connectionstring)).toThrowError('Choose between apikey, token or username/password') @@ -179,6 +186,19 @@ describe('parseconnectionstring', () => { }) }) +describe('validateConfiguration()', () => { + it('should use safe integer mode from config', () => { + const config = validateConfiguration({ + username: 'user', + password: 'password', + host: 'host', + safe_integer_mode: 'mixed' + }) + + expect(config.safe_integer_mode).toBe('mixed') + }) +}) + describe('getTestingDatabaseName', () => { it('should generate readable database names', () => { const database = getTestingDatabaseName('benchkmark') @@ -218,4 +238,4 @@ describe('getInitializationCommands()', () => { expect(result).toContain('AUTH TOKEN mytoken;') expect(result).not.toContain('AUTH APIKEY') }) -}) \ No newline at end of file +})