feat: Add support for cds.Vector on SQLite and Postgres

hana/lib/cql-functions.js

@@ -267,6 +267,15 @@ const HANAFunctions = {
   HIERARCHY: undefined,
   HIERARCHY_DESCENDANTS: undefined,
   HIERARCHY_ANCESTORS: undefined,
+
+  // Ease of use for stakeholders for auto-filling the remote source
+  vector_embedding(text, text_type, model_and_version, remote_source = cds.env.ai?.embeddings?.remoteSource) {
+    if (!remote_source || (model_and_version.val ?? model_and_version).startswith('SAP')) {
+      return `vector_embedding(${text},${text_type},${model_and_version})`
+    } else {
+      return `vector_embedding(${text},${text_type},${model_and_version},${remote_source})`
+    }
+  }
 }
 
 for (let each in HANAFunctions) HANAFunctions[each.toUpperCase()] = HANAFunctions[each]

postgres/lib/PostgresService.js

@@ -4,6 +4,7 @@ const cds = require('@sap/cds')
 const crypto = require('crypto')
 const { Writable, Readable } = require('stream')
 const sessionVariableMap = require('./session.json')
+const pgvector = require('pgvector/pg');
 
 const LOG = cds.log('sql|db')
 
@@ -49,6 +50,14 @@ class PostgresService extends SQLService {
         try {
           const dbc = new Client({ ...credentials, ...clientOptions })
           await dbc.connect()
+          // cds.Vector support for PG
+          try {
+            await dbc.query('CREATE EXTENSION IF NOT EXISTS vector')
+            await pgvector.registerTypes(dbc)
+          } catch (e) {
+            const LOG = cds.log('postgres')
+            LOG.debug('pgvector extension not available, skipping vector support:', e.message)
+          }
           dbc.open = true
           dbc.on('end', () => { dbc.open = false })
           return dbc
@@ -535,6 +544,7 @@ GROUP BY k
       DateTime: () => 'TIMESTAMP',
       Timestamp: () => 'TIMESTAMP',
       Map: () => 'JSONB',
+      Vector: () => 'vector',
 
       // HANA Types
       'cds.hana.CLOB': () => 'BYTEA',

postgres/lib/cql-functions.js

@@ -191,6 +191,33 @@ const HANAFunctions = {
   years_between(x, y) {
     return `TRUNC(${this.expr({ func: 'months_between', args: [x, y] })} / 12,0)`
   },
+
+  /**
+   * Computes the cosine similarity of two vectors
+   * @param {*} v1 - Vector 1
+   * @param {*} v2 - Vector 2
+   * @returns {string} - SQL statement
+   */
+  cosine_similarity(v1, v2) {
+    return `1 - cosine_distance(${this.expr(v1)},${this.expr(v2)})`
+  },
+  /**
+   * Computes the L2 distance of two vectors.
+   * @param {*} v1 - Vector 1
+   * @param {*} v2 - Vector 2
+   * @returns {string} - SQL statement
+   */
+  l2distance(v1, v2) { 
+    return `l2_distance(${this.expr(v1)},${this.expr(v2)})`
+  },
+  /**
+   * L2 normalizes the vector
+   * @param {*} v - Vector
+   * @returns {string} - SQL statement
+   */
+  l2normalize(v) { 
+    return `l2_normalize(${this.expr(v)})`
+  },
 }
 
 for (let each in HANAFunctions) HANAFunctions[each.toUpperCase()] = HANAFunctions[each]

postgres/package.json

@@ -28,7 +28,8 @@
   },
   "dependencies": {
     "@cap-js/db-service": "^3.0.1",
-    "pg": "^8"
+    "pg": "^8",
+    "pgvector": "^0.2.1"
   },
   "peerDependencies": {
     "@sap/cds": "^10",

postgres/pg-stack.yml

@@ -3,10 +3,13 @@ version: '3.1'
 
 services:
   db:
-    image: postgres:16-alpine
+    image: pgvector/pgvector:pg16
     restart: always
     environment:
       POSTGRES_PASSWORD: postgres
+      LANG: en_US.utf8
+      LC_COLLATE: en_US.utf8
+      LC_CTYPE: en_US.utf8
     ports:
       - '5432:5432'
     command: ['postgres', '-c', 'log_statement=all']

sqlite/lib/SQLiteService.js

@@ -6,6 +6,7 @@ const $session = Symbol('dbc.session')
 const sessionVariableMap = require('./session.json')  // Adjust the path as necessary for your project
 const convStrm = require('stream/consumers')
 const { Readable } = require('stream')
+const { cosineSimilarity, l2Distance, l2Normalize, hashEmbedding, toFloatArray, fromFloatArray } = require('./vector_handling')
 
 const keywords = cds.compiler.to.sql.sqlite.keywords
 // keywords come as array
@@ -46,6 +47,10 @@ class SQLiteService extends SQLService {
           dbc.function('hour', deterministic, d => d === null ? null : toDate(d, true).getUTCHours())
           dbc.function('minute', deterministic, d => d === null ? null : toDate(d, true).getUTCMinutes())
           dbc.function('second', deterministic, d => d === null ? null : toDate(d, true).getUTCSeconds())
+          dbc.function('COSINE_SIMILARITY', deterministic, (a, b) => cosineSimilarity(toFloatArray(a), toFloatArray(b)))
+          dbc.function('L2DISTANCE', deterministic, (a, b) => l2Distance(toFloatArray(a), toFloatArray(b)))
+          dbc.function('L2NORMALIZE', deterministic, v => v == null ? null : fromFloatArray(l2Normalize(toFloatArray(v)), v))
+          dbc.function('VECTOR_EMBEDDING', deterministic, (model, text) => text == null ? null : JSON.stringify(hashEmbedding(String(text))))
           if (database !== ':memory:') dbc.pragma?.('journal_mode = WAL') || dbc.exec('PRAGMA journal_mode = WAL')
           return dbc
         } catch (err) {

sqlite/lib/vector_handling/index.js

@@ -0,0 +1,92 @@
+function cosineSimilarity(a, b) {
+  if (a == null || b == null) return null
+  let dot = 0, normA = 0, normB = 0
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i]
+    normA += a[i] * a[i]
+    normB += b[i] * b[i]
+  }
+  const denom = Math.sqrt(normA) * Math.sqrt(normB)
+  return denom === 0 ? 0 : dot / denom
+}
+
+function l2Distance(a, b) {
+  if (a == null || b == null) return null
+  let sum = 0
+  for (let i = 0; i < a.length; i++) {
+    const diff = a[i] - b[i]
+    sum += diff * diff
+  }
+  return Math.sqrt(sum)
+}
+
+function l2Normalize(v) {
+  if (v == null) return null
+  let norm = 0
+  for (let i = 0; i < v.length; i++) norm += v[i] * v[i]
+  if (norm === 0) return v
+  norm = Math.sqrt(norm)
+  for (let i = 0; i < v.length; i++) v[i] /= norm
+  return v
+}
+
+/**
+ * Deterministic synchronous hash-based embedding function for SQLite.
+ * NOTE: If a synchronous embedding library becomes available for Node.js,
+ * it can be integrated here to replace the hash-based implementation.
+ */
+function hashEmbedding(text, dimensions = 384) {
+  if (text == null) return null
+  const vector = new Float32Array(dimensions)
+  const normalized = text.toLowerCase()
+  const ngramSize = 3
+
+  if (normalized.length >= ngramSize) {
+    for (let i = 0; i <= normalized.length - ngramSize; i++)
+      project(ngramHash(normalized, i, ngramSize), vector, dimensions)
+  } else {
+    for (let i = 0; i < normalized.length; i++)
+      project(normalized.charCodeAt(i), vector, dimensions)
+  }
+  return Array.from(l2Normalize(vector))
+}
+
+function ngramHash(text, start, len) {
+  let hash = 0x811c9dc5
+  for (let i = start; i < start + len; i++) {
+    hash ^= text.charCodeAt(i)
+    hash = Math.imul(hash, 0x01000193)
+  }
+  return hash
+}
+
+function project(hash, vector, dimensions) {
+  for (let band = 0; band < 4; band++) {
+    const h = rehash(hash, band)
+    vector[Math.abs(h % dimensions)] += ((h >>> 16) & 1) === 0 ? 1.0 : -1.0
+  }
+}
+
+function rehash(hash, band) {
+  let h = hash ^ Math.imul(band, 0x9e3779b9)
+  h ^= h >>> 16
+  h = Math.imul(h, 0x45d9f3b)
+  h ^= h >>> 16
+  return h
+}
+
+function toFloatArray(vector) {
+  if (vector == null) return null
+  if (vector instanceof Float32Array) return Array.from(vector)
+  if (Buffer.isBuffer(vector)) return JSON.parse(vector.toString('utf8'))
+  if (vector instanceof Uint8Array) return JSON.parse(Buffer.from(vector).toString('utf8'))
+  if (typeof vector === 'string') return JSON.parse(vector)
+  if (Array.isArray(vector)) return vector
+  throw new Error(`Unsupported vector type: ${typeof vector}`)
+}
+
+function fromFloatArray(arr, original) {
+  return original instanceof Float32Array ? new Float32Array(arr) : JSON.stringify(arr)
+}
+
+module.exports = { cosineSimilarity, l2Distance, l2Normalize, hashEmbedding, toFloatArray, fromFloatArray }

test/compliance/functions.test.js

@@ -1,5 +1,7 @@
 const cds = require('../cds.js')
 
+const isHana = () => cds.db?.options?.impl === '@cap-js/hana'
+
 describe('functions', () => {
   const { expect, data } = cds.test(__dirname + '/resources')
   data.autoIsolation(true)
@@ -229,6 +231,26 @@ describe('functions', () => {
       throw new Error('not supported')
     })
   })
+  describe('COSINE_SIMILARITY', () => {
+    test('identical vectors return 1', async () => {
+      if (isHana()) return // HANA Express doesn't support REAL_VECTOR
+      const res = await SELECT.from('complex.associations.Books')
+        .columns`cosine_similarity(cast('[1, 0, 0]' as cds.Vector), cast('[1, 0, 0]' as cds.Vector)) as similarity`
+      expect(res[0].similarity).to.eq(1)
+    })
+    test('orthogonal vectors return 0', async () => {
+      if (isHana()) return
+      const res = await SELECT.from('complex.associations.Books')
+        .columns`cosine_similarity(cast('[1, 0, 0]' as cds.Vector), cast('[0, 1, 0]' as cds.Vector)) as similarity`
+      expect(res[0].similarity).to.eq(0)
+    })
+    test('opposite vectors return -1', async () => {
+      if (isHana()) return
+      const res = await SELECT.from('complex.associations.Books')
+        .columns`cosine_similarity(cast('[1, 0, 0]' as cds.Vector), cast('[-1, 0, 0]' as cds.Vector)) as similarity`
+      expect(res[0].similarity).to.eq(-1)
+    })
+  })
   describe('COSH', () => {
     test.skip('missing', () => {
       throw new Error('not supported')
@@ -535,6 +557,45 @@ describe('functions', () => {
       throw new Error('not supported')
     })
   })
+  describe('L2DISTANCE', () => {
+    test('identical vectors return 0', async () => {
+      if (isHana()) return
+      const res = await SELECT.from('complex.associations.Books')
+        .columns`l2distance(cast('[1, 0, 0]' as cds.Vector), cast('[1, 0, 0]' as cds.Vector)) as distance`
+      expect(res[0].distance).to.eq(0)
+    })
+    test('unit vectors distance', async () => {
+      if (isHana()) return
+      const res = await SELECT.from('complex.associations.Books')
+        .columns`l2distance(cast('[1, 0, 0]' as cds.Vector), cast('[0, 1, 0]' as cds.Vector)) as distance`
+      expect(Math.abs(res[0].distance - Math.SQRT2) < 0.0001).to.eq(true)
+    })
+    test('known distance', async () => {
+      if (isHana()) return
+      const res = await SELECT.from('complex.associations.Books')
+        .columns`l2distance(cast('[0, 0, 0]' as cds.Vector), cast('[3, 4, 0]' as cds.Vector)) as distance`
+      expect(res[0].distance).to.eq(5)
+    })
+  })
+  describe('L2NORMALIZE', () => {
+    test('normalizes to unit length', async () => {
+      if (isHana()) return
+      const res = await SELECT.from('complex.associations.Books')
+        .columns`l2normalize(cast('[3, 4, 0]' as cds.Vector)) as normalized`
+      const normalized = JSON.parse(res[0].normalized)
+      expect(Math.abs(normalized[0] - 0.6) < 0.0001).to.eq(true)
+      expect(Math.abs(normalized[1] - 0.8) < 0.0001).to.eq(true)
+      expect(normalized[2]).to.eq(0)
+    })
+    test('already normalized unchanged', async () => {
+      if (isHana()) return
+      const res = await SELECT.from('complex.associations.Books')
+        .columns`l2normalize(cast('[1, 0, 0]' as cds.Vector)) as normalized`
+      const normalized = JSON.parse(res[0].normalized)
+      expect(normalized[0]).to.eq(1)
+      expect(normalized[1]).to.eq(0)
+    })
+  })
   describe('LAG', () => {
     test.skip('missing', () => {
       throw new Error('not supported')
@@ -1213,6 +1274,35 @@ describe('functions', () => {
       throw new Error('not supported')
     })
   })
+  describe('VECTOR_EMBEDDING', () => {
+    test('computes embedding', async () => {
+      if (isHana()) return
+      const res = await SELECT.from('complex.associations.Books')
+        .columns`VECTOR_EMBEDDING('model', title) as embedding`
+        .limit(1)
+      const embedding = JSON.parse(res[0].embedding)
+      expect(Array.isArray(embedding)).to.eq(true)
+      expect(embedding.length).to.eq(384)
+    })
+    test('deterministic', async () => {
+      if (isHana()) return
+      const res = await SELECT.from('complex.associations.Books')
+        .columns`vector_embedding('model', 'test') as e1, vector_embedding('model', 'test') as e2`
+      expect(res[0].e1).to.eq(res[0].e2)
+    })
+    test('different inputs different outputs', async () => {
+      if (isHana()) return
+      const res = await SELECT.from('complex.associations.Books')
+        .columns`vector_embedding('model', 'hello') as e1, vector_embedding('model', 'world') as e2`
+      expect(res[0].e1).to.not.eq(res[0].e2)
+    })
+    test('null returns null', async () => {
+      if (isHana()) return
+      const res = await SELECT.from('complex.associations.Books')
+        .columns`vector_embedding('model', null) as embedding`
+      expect(res[0].embedding).to.eq(null)
+    })
+  })
   describe('WEEK', () => {
     test.skip('missing', () => {
       throw new Error('not supported')