feat: Add support for cds.Vector on SQLite and Postgres

hana/lib/cql-functions.js

@@ -267,6 +267,15 @@ const HANAFunctions = {
   HIERARCHY: undefined,
   HIERARCHY_DESCENDANTS: undefined,
   HIERARCHY_ANCESTORS: undefined,
+
+  // Ease of use for stakeholders for auto-filling the remote source
+  vector_embedding(text, text_type, model_and_version, remote_source = cds.env.ai?.embeddings?.remoteSource) {
+    if (!remote_source || (model_and_version.val ?? model_and_version).startswith('SAP')) {
+      return `vector_embedding(${text},${text_type},${model_and_version})`
+    } else {
+      return `vector_embedding(${text},${text_type},${model_and_version},${remote_source})`
+    }
+  }
 }
 
 for (let each in HANAFunctions) HANAFunctions[each.toUpperCase()] = HANAFunctions[each]

postgres/lib/PostgresService.js

@@ -49,6 +49,22 @@ class PostgresService extends SQLService {
         try {
           const dbc = new Client({ ...credentials, ...clientOptions })
           await dbc.connect()
+          // cds.Vector support for PG
+          // REVISIT: Check if module 'pgvector/pg' is installed
+          // More ideas:
+          // 1. Move CREATE EXTENSION to deployment (migration/deployment scripts). The CDS build/deploy could handle this
+          // 2. Only pgvector.registerTypes(dbc) - This registers type parsers with the pg client and may need to run per connection
+          // 3. Make it conditional - Only run if a config flag is set, e.g., cds.env.requires.db.vector: true
+          // 4. Lazy initialization - Only initialize vector support when a vector operation is first attempted
+          // 5. Run once per process - Use a module-level flag         
+          /*
+          try {
+            await dbc.query('CREATE EXTENSION IF NOT EXISTS vector')
+            await pgvector.registerTypes(dbc)
+          } catch (e) {
+            const LOG = cds.log('postgres')
+            LOG.debug('pgvector extension not available, skipping vector support:', e.message)
+          }*/
           dbc.open = true
           dbc.on('end', () => { dbc.open = false })
           return dbc
@@ -89,7 +105,7 @@ class PostgresService extends SQLService {
         JSON.stringify(env),
       ]),
       ...(this.options?.credentials?.schema
-        ? [this.exec(`SET search_path TO "${this.options?.credentials?.schema}";`)]
+        ? [this.exec(`SET search_path TO "${this.options?.credentials?.schema}", public;`)] // include public for extensions like pgvector
         : []),
 
       ...(!this._initalCollateCheck ? [this._checkCollation()] : []),
@@ -535,6 +551,7 @@ GROUP BY k
       DateTime: () => 'TIMESTAMP',
       Timestamp: () => 'TIMESTAMP',
       Map: () => 'JSONB',
+      Vector: () => 'vector',
 
       // HANA Types
       'cds.hana.CLOB': () => 'BYTEA',

postgres/lib/cql-functions.js

@@ -191,6 +191,33 @@ const HANAFunctions = {
   years_between(x, y) {
     return `TRUNC(${this.expr({ func: 'months_between', args: [x, y] })} / 12,0)`
   },
+
+  /**
+   * Computes the cosine similarity of two vectors
+   * @param {*} v1 - Vector 1
+   * @param {*} v2 - Vector 2
+   * @returns {string} - SQL statement
+   */
+  cosine_similarity(v1, v2) {
+    return `1 - cosine_distance(${this.expr(v1)},${this.expr(v2)})`
+  },
+  /**
+   * Computes the L2 distance of two vectors.
+   * @param {*} v1 - Vector 1
+   * @param {*} v2 - Vector 2
+   * @returns {string} - SQL statement
+   */
+  l2distance(v1, v2) { 
+    return `l2_distance(${this.expr(v1)},${this.expr(v2)})`
+  },
+  /**
+   * L2 normalizes the vector
+   * @param {*} v - Vector
+   * @returns {string} - SQL statement
+   */
+  l2normalize(v) { 
+    return `l2_normalize(${this.expr(v)})`
+  },
 }
 
 for (let each in HANAFunctions) HANAFunctions[each.toUpperCase()] = HANAFunctions[each]

postgres/package.json

@@ -30,6 +30,9 @@
     "@cap-js/db-service": "^3.0.1",
     "pg": "^8.22.0"
   },
+  "devDependencies": {
+    "pgvector": "^0.3.0"
+  },
   "peerDependencies": {
     "@sap/cds": "^10",
     "@sap/cds-dk": "^10"

postgres/pg-stack.yml

@@ -1,9 +1,7 @@
 # Use postgres/example user/password credentials
-version: '3.1'
-
 services:
   db:
-    image: postgres:16-alpine
+    image: pgvector/pgvector:pg16
     restart: always
     environment:
       POSTGRES_PASSWORD: postgres

postgres/test/vector.test.js

@@ -0,0 +1,132 @@
+const cds = require('../../test/cds.js')
+const { Client } = require('pg')
+const os = require('os')
+
+describe('vector functions', () => {
+  const { expect } = cds.test(__dirname + '/../../test/compliance/resources')
+
+  // Setup pgvector extension and fake vector_embedding function
+  beforeAll(async () => {
+    const testDb = process.env.TRAVIS_JOB_ID || process.env.GITHUB_RUN_ID || os.userInfo().username || 'test_db'
+
+    // First connect to postgres db to create extension in template1 and ensure test db exists
+    const adminClient = new Client({
+      host: 'localhost',
+      port: 5432,
+      user: 'postgres',
+      password: 'postgres',
+      database: 'postgres'
+    })
+    await adminClient.connect()
+    await adminClient.query('CREATE EXTENSION IF NOT EXISTS vector')
+
+    // Create test database if it doesn't exist
+    try {
+      await adminClient.query(`CREATE DATABASE "${testDb}"`)
+    } catch (e) {
+      // Database might already exist
+    }
+    await adminClient.end()
+
+    // Now connect to the test database and set up extension + function
+    const client = new Client({
+      host: 'localhost',
+      port: 5432,
+      user: 'postgres',
+      password: 'postgres',
+      database: testDb
+    })
+    await client.connect()
+    await client.query('CREATE EXTENSION IF NOT EXISTS vector')
+    await client.query(`
+      CREATE OR REPLACE FUNCTION public.vector_embedding(model text, input text)
+      RETURNS text AS $$
+        SELECT CASE WHEN input IS NULL THEN NULL
+          ELSE (SELECT json_agg(sin(i * hashtext(input)::float8 / 1000))::text
+                FROM generate_series(1, 384) i)
+        END;
+      $$ LANGUAGE SQL IMMUTABLE;
+    `)
+    await client.end()
+  })
+
+  describe('COSINE_SIMILARITY', () => {
+    test('identical vectors return 1', async () => {
+      const res = await SELECT.from('complex.associations.Books')
+        .columns`cosine_similarity(cast('[1, 0, 0]' as cds.Vector), cast('[1, 0, 0]' as cds.Vector)) as similarity`
+      expect(res[0].similarity).to.eq(1)
+    })
+    test('orthogonal vectors return 0', async () => {
+      const res = await SELECT.from('complex.associations.Books')
+        .columns`cosine_similarity(cast('[1, 0, 0]' as cds.Vector), cast('[0, 1, 0]' as cds.Vector)) as similarity`
+      expect(res[0].similarity).to.eq(0)
+    })
+    test('opposite vectors return -1', async () => {
+      const res = await SELECT.from('complex.associations.Books')
+        .columns`cosine_similarity(cast('[1, 0, 0]' as cds.Vector), cast('[-1, 0, 0]' as cds.Vector)) as similarity`
+      expect(res[0].similarity).to.eq(-1)
+    })
+  })
+
+  describe('L2DISTANCE', () => {
+    test('identical vectors return 0', async () => {
+      const res = await SELECT.from('complex.associations.Books')
+        .columns`l2distance(cast('[1, 0, 0]' as cds.Vector), cast('[1, 0, 0]' as cds.Vector)) as distance`
+      expect(res[0].distance).to.eq(0)
+    })
+    test('unit vectors distance', async () => {
+      const res = await SELECT.from('complex.associations.Books')
+        .columns`l2distance(cast('[1, 0, 0]' as cds.Vector), cast('[0, 1, 0]' as cds.Vector)) as distance`
+      expect(Math.abs(res[0].distance - Math.SQRT2) < 0.0001).to.eq(true)
+    })
+    test('known distance', async () => {
+      const res = await SELECT.from('complex.associations.Books')
+        .columns`l2distance(cast('[0, 0, 0]' as cds.Vector), cast('[3, 4, 0]' as cds.Vector)) as distance`
+      expect(res[0].distance).to.eq(5)
+    })
+  })
+
+  describe('L2NORMALIZE', () => {
+    test('normalizes to unit length', async () => {
+      const res = await SELECT.from('complex.associations.Books')
+        .columns`l2normalize(cast('[3, 4, 0]' as cds.Vector)) as normalized`
+      const normalized = JSON.parse(res[0].normalized)
+      expect(Math.abs(normalized[0] - 0.6) < 0.0001).to.eq(true)
+      expect(Math.abs(normalized[1] - 0.8) < 0.0001).to.eq(true)
+      expect(normalized[2]).to.eq(0)
+    })
+    test('already normalized unchanged', async () => {
+      const res = await SELECT.from('complex.associations.Books')
+        .columns`l2normalize(cast('[1, 0, 0]' as cds.Vector)) as normalized`
+      const normalized = JSON.parse(res[0].normalized)
+      expect(normalized[0]).to.eq(1)
+      expect(normalized[1]).to.eq(0)
+    })
+  })
+
+  describe('VECTOR_EMBEDDING', () => {
+    test('computes embedding', async () => {
+      const res = await SELECT.from('complex.associations.Books')
+        .columns`VECTOR_EMBEDDING('model', title) as embedding`
+        .limit(1)
+      const embedding = JSON.parse(res[0].embedding)
+      expect(Array.isArray(embedding)).to.eq(true)
+      expect(embedding.length).to.eq(384)
+    })
+    test('deterministic', async () => {
+      const res = await SELECT.from('complex.associations.Books')
+        .columns`vector_embedding('model', 'test') as e1, vector_embedding('model', 'test') as e2`
+      expect(res[0].e1).to.eq(res[0].e2)
+    })
+    test('different inputs different outputs', async () => {
+      const res = await SELECT.from('complex.associations.Books')
+        .columns`vector_embedding('model', 'hello') as e1, vector_embedding('model', 'world') as e2`
+      expect(res[0].e1).to.not.eq(res[0].e2)
+    })
+    test('null returns null', async () => {
+      const res = await SELECT.from('complex.associations.Books')
+        .columns`vector_embedding('model', null) as embedding`
+      expect(res[0].embedding).to.eq(null)
+    })
+  })
+})