From 4cf12f45ac5063d998ada5708452c3fb0da4d598 Mon Sep 17 00:00:00 2001 From: Michel Albers Date: Wed, 15 Jun 2022 10:37:03 +0200 Subject: [PATCH 1/2] feat: allow nested filtering --- lib/pubsub/getFilteredSubs-test.ts | 195 +++++++++++++++++++---------- lib/pubsub/getFilteredSubs.ts | 107 ++++++++++------ 2 files changed, 196 insertions(+), 106 deletions(-) diff --git a/lib/pubsub/getFilteredSubs-test.ts b/lib/pubsub/getFilteredSubs-test.ts index 689f447a..2da2770b 100644 --- a/lib/pubsub/getFilteredSubs-test.ts +++ b/lib/pubsub/getFilteredSubs-test.ts @@ -1,103 +1,166 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ -import { tables } from '@architect/sandbox' -import { assert } from 'chai' -import { mockServerContext } from '../test/mockServer' -import { collapseKeys, getFilteredSubs } from './getFilteredSubs' - -describe('collapseKeys', () => { - it('makes the deep objects into dots', () => { - assert.deepEqual(collapseKeys({}), {}) - assert.deepEqual(collapseKeys({ a: 4, b: { c: 5, d: 'hi', e: { f: false } } }), { - a: 4, - 'b.c': 5, - 'b.d': 'hi', - 'b.e.f': false, - }) - assert.deepEqual(collapseKeys({ a: [1, 2, 3, { b: 4, c: [], d: null, e: undefined }] }), { - 'a.0': 1, - 'a.1': 2, - 'a.2': 3, - 'a.3.b': 4, - }) - }) -}) +import { tables } from "@architect/sandbox"; +import { assert } from "chai"; +import { mockServerContext } from "../test/mockServer"; +import { collapseKeys, getFilteredSubs } from "./getFilteredSubs"; +describe("collapseKeys", () => { + it("makes the deep objects into dots", () => { + assert.deepEqual(collapseKeys({}), {}); + assert.deepEqual( + collapseKeys({ a: 4, b: { c: 5, d: "hi", e: { f: false } } }), + { + a: 4, + "b.c": 5, + "b.d": "hi", + "b.e.f": false, + } + ); + assert.deepEqual( + collapseKeys({ a: [1, 2, 3, { b: 4, c: [], d: null, e: undefined }] }), + { + "a.0": 1, + "a.1": 2, + "a.2": 3, + "a.3.b": 4, + } + ); + }); +}); // since we're not resetting the db every time we need to change this -let count = 1 -const makeTopic = () => `topic-${count++}` +let count = 1; +const makeTopic = () => `topic-${count++}`; -describe('getFilteredSubs', () => { +describe("getFilteredSubs", () => { before(async () => { - await tables.start({ cwd: './mocks/arc-basic-events', quiet: true }) - }) + await tables.start({ cwd: "./mocks/arc-basic-events", quiet: true }); + }); after(async () => { - await tables.end() - }) + await tables.end(); + }); - it('can match on no filter', async () => { - const topic = makeTopic() - const server = await mockServerContext() + it("can match on no filter", async () => { + const topic = makeTopic(); + const server = await mockServerContext(); const subscription = { - id: '1', + id: "1", topic, filter: {}, - subscriptionId: '1', + subscriptionId: "1", subscription: {} as any, - connectionId: 'abcd', + connectionId: "abcd", connectionInitPayload: {}, requestContext: {} as any, ttl: Math.floor(Date.now() / 1000) + 100000, createdAt: Date.now(), - } + }; + + await server.models.subscription.put(subscription); + assert.containSubset( + await getFilteredSubs({ + server, + event: { topic, payload: { language: "en" } }, + }), + [{ topic, id: "1" }] + ); + }); + + it("can match on payload", async () => { + const topic = makeTopic(); + const server = await mockServerContext(); + const subscription = { + id: "2", + topic, + filter: { language: "en" }, + subscriptionId: "2", + subscription: {} as any, + connectionId: "abcd", + connectionInitPayload: {}, + requestContext: {} as any, + ttl: Math.floor(Date.now() / 1000) + 100000, + createdAt: Date.now(), + }; + + await server.models.subscription.put(subscription); - await server.models.subscription.put(subscription) - assert.containSubset(await getFilteredSubs({ server, event: { topic, payload: { language: 'en' } } }), [{ topic, id: '1' }]) - }) + assert.containSubset( + await getFilteredSubs({ + server, + event: { topic, payload: { language: "en" } }, + }), + [{ topic, id: "2" }] + ); + assert.deepEqual( + await getFilteredSubs({ + server, + event: { topic, payload: { language: "en-gb" } }, + }), + [] + ); + }); - it('can match on payload', async () => { - const topic = makeTopic() - const server = await mockServerContext() + it("can match on nested payload", async () => { + const topic = makeTopic(); + const server = await mockServerContext(); const subscription = { - id: '2', + id: "2", topic, - filter: { language: 'en' }, - subscriptionId: '2', + filter: { meta: { user: "foo" }, message: { content: "hi" } }, + subscriptionId: "2", subscription: {} as any, - connectionId: 'abcd', + connectionId: "abcd", connectionInitPayload: {}, requestContext: {} as any, ttl: Math.floor(Date.now() / 1000) + 100000, createdAt: Date.now(), - } + }; - await server.models.subscription.put(subscription) + await server.models.subscription.put(subscription); - assert.containSubset(await getFilteredSubs({ server, event: { topic, payload: { language: 'en' } } }), [{ topic, id: '2' }]) - assert.deepEqual(await getFilteredSubs({ server, event: { topic, payload: { language: 'en-gb' } } }), []) - }) + assert.containSubset( + await getFilteredSubs({ + server, + event: { + topic, + payload: { meta: { user: "foo" }, message: { content: "hi" } }, + }, + }), + [{ topic, id: "2" }] + ); + assert.deepEqual( + await getFilteredSubs({ + server, + event: { + topic, + payload: { meta: { user: "lol" }, message: { content: "bye" } }, + }, + }), + [] + ); + }); - it('can match on no payload', async () => { - const topic = makeTopic() - const server = await mockServerContext() + it("can match on no payload", async () => { + const topic = makeTopic(); + const server = await mockServerContext(); const subscription = { - id: '1234567', + id: "1234567", topic, - filter: { language: 'en ' }, - subscriptionId: '12345', + filter: { language: "en " }, + subscriptionId: "12345", subscription: {} as any, - connectionId: '1234', + connectionId: "1234", connectionInitPayload: {}, requestContext: {} as any, ttl: Math.floor(Date.now() / 1000) + 100000, createdAt: Date.now(), - } + }; - await server.models.subscription.put(subscription) - const subscriptions = await getFilteredSubs({ server, event: { topic } }) - assert.containSubset(subscriptions, [{ topic, id: '1234567' }]) - }) - it('can match on connectionId') - it('can match on topic key') -}) + await server.models.subscription.put(subscription); + const subscriptions = await getFilteredSubs({ server, event: { topic } }); + assert.containSubset(subscriptions, [{ topic, id: "1234567" }]); + }); + it("can match on connectionId"); + it("can match on topic key"); +}); diff --git a/lib/pubsub/getFilteredSubs.ts b/lib/pubsub/getFilteredSubs.ts index ccf7c0aa..558745b3 100644 --- a/lib/pubsub/getFilteredSubs.ts +++ b/lib/pubsub/getFilteredSubs.ts @@ -1,75 +1,102 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ -import { collect } from 'streaming-iterables' -import { ServerClosure, Subscription } from '../types' +import { collect } from "streaming-iterables"; +import { ServerClosure, Subscription } from "../types"; -export const getFilteredSubs = async ({ server, event }: { server: Omit, event: { topic: string, payload?: Record } }): Promise => { +export const getFilteredSubs = async ({ + server, + event, +}: { + server: Omit; + event: { topic: string; payload?: Record }; +}): Promise => { if (!event.payload || Object.keys(event.payload).length === 0) { - server.log('getFilteredSubs', { event }) + server.log("getFilteredSubs", { event }); const iterator = server.models.subscription.query({ - IndexName: 'TopicIndex', - ExpressionAttributeNames: { '#a': 'topic' }, - ExpressionAttributeValues: { ':1': event.topic }, - KeyConditionExpression: '#a = :1', - }) + IndexName: "TopicIndex", + ExpressionAttributeNames: { "#a": "topic" }, + ExpressionAttributeValues: { ":1": event.topic }, + KeyConditionExpression: "#a = :1", + }); - return await collect(iterator) + return await collect(iterator); } - const flattenPayload = collapseKeys(event.payload) + const flattenPayload = collapseKeys(event.payload); - const filterExpressions: string[] = [] - const expressionAttributeValues: { [key: string]: string | number | boolean } = {} - const expressionAttributeNames: { [key: string]: string } = {} + const filterExpressions: string[] = []; + const expressionAttributeValues: { + [key: string]: string | number | boolean; + } = {}; + const expressionAttributeNames: { [key: string]: string } = {}; - let attributeCounter = 0 + let attributeCounter = 0; for (const [key, value] of Object.entries(flattenPayload)) { - const aliasNumber = attributeCounter++ - expressionAttributeNames[`#${aliasNumber}`] = key - expressionAttributeValues[`:${aliasNumber}`] = value - filterExpressions.push(`(#filter.#${aliasNumber} = :${aliasNumber} OR attribute_not_exists(#filter.#${aliasNumber}))`) + let aliasNumber = attributeCounter++; + const keyParts = key.split("."); + const keyPartsAttributeName = keyParts + .map((part, index) => `#${aliasNumber + index}`) + .join("."); + key.split(".").forEach((keyPart, index) => { + expressionAttributeNames[`#${aliasNumber + index}`] = keyPart; + attributeCounter += index; + }); + expressionAttributeValues[`:${aliasNumber}`] = value; + + filterExpressions.push( + `(#filter.${keyPartsAttributeName} = :${aliasNumber} OR attribute_not_exists(#filter.${keyPartsAttributeName}))` + ); } - server.log('getFilteredSubs', { event, expressionAttributeNames, expressionAttributeValues, filterExpressions }) + server.log("getFilteredSubs", { + event, + expressionAttributeNames, + expressionAttributeValues, + filterExpressions, + }); const iterator = server.models.subscription.query({ - IndexName: 'TopicIndex', + IndexName: "TopicIndex", ExpressionAttributeNames: { - '#hashKey': 'topic', - '#filter': 'filter', + "#hashKey": "topic", + "#filter": "filter", ...expressionAttributeNames, }, ExpressionAttributeValues: { - ':hashKey': event.topic, + ":hashKey": event.topic, ...expressionAttributeValues, }, - KeyConditionExpression: '#hashKey = :hashKey', - FilterExpression: filterExpressions.join(' AND ') || undefined, - }) + KeyConditionExpression: "#hashKey = :hashKey", + FilterExpression: filterExpressions.join(" AND ") || undefined, + }); - return await collect(iterator) -} + return await collect(iterator); +}; export const collapseKeys = ( - obj: Record, + obj: Record ): Record => { - const record = {} + const record = {}; for (const [k1, v1] of Object.entries(obj)) { - if (typeof v1 === 'string' || typeof v1 === 'number' || typeof v1 === 'boolean') { - record[k1] = v1 - continue + if ( + typeof v1 === "string" || + typeof v1 === "number" || + typeof v1 === "boolean" + ) { + record[k1] = v1; + continue; } - if (v1 && typeof v1 === 'object') { - const next = {} + if (v1 && typeof v1 === "object") { + const next = {}; for (const [k2, v2] of Object.entries(v1)) { - next[`${k1}.${k2}`] = v2 + next[`${k1}.${k2}`] = v2; } for (const [k1, v1] of Object.entries(collapseKeys(next))) { - record[k1] = v1 + record[k1] = v1; } } } - return record -} + return record; +}; From 066f1de579eb434d924f943a1621b0e4f84b82ae Mon Sep 17 00:00:00 2001 From: Michel Albers Date: Wed, 15 Jun 2022 10:38:25 +0200 Subject: [PATCH 2/2] chore: remove white space changes --- lib/pubsub/getFilteredSubs-test.ts | 174 ++++++++++++++--------------- lib/pubsub/getFilteredSubs.ts | 102 ++++++++--------- 2 files changed, 138 insertions(+), 138 deletions(-) diff --git a/lib/pubsub/getFilteredSubs-test.ts b/lib/pubsub/getFilteredSubs-test.ts index 2da2770b..231c25b9 100644 --- a/lib/pubsub/getFilteredSubs-test.ts +++ b/lib/pubsub/getFilteredSubs-test.ts @@ -1,166 +1,166 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ -import { tables } from "@architect/sandbox"; -import { assert } from "chai"; -import { mockServerContext } from "../test/mockServer"; -import { collapseKeys, getFilteredSubs } from "./getFilteredSubs"; +import { tables } from '@architect/sandbox' +import { assert } from 'chai' +import { mockServerContext } from '../test/mockServer' +import { collapseKeys, getFilteredSubs } from './getFilteredSubs' -describe("collapseKeys", () => { - it("makes the deep objects into dots", () => { - assert.deepEqual(collapseKeys({}), {}); +describe('collapseKeys', () => { + it('makes the deep objects into dots', () => { + assert.deepEqual(collapseKeys({}), {}) assert.deepEqual( - collapseKeys({ a: 4, b: { c: 5, d: "hi", e: { f: false } } }), + collapseKeys({ a: 4, b: { c: 5, d: 'hi', e: { f: false } } }), { a: 4, - "b.c": 5, - "b.d": "hi", - "b.e.f": false, - } - ); + 'b.c': 5, + 'b.d': 'hi', + 'b.e.f': false, + }, + ) assert.deepEqual( collapseKeys({ a: [1, 2, 3, { b: 4, c: [], d: null, e: undefined }] }), { - "a.0": 1, - "a.1": 2, - "a.2": 3, - "a.3.b": 4, - } - ); - }); -}); + 'a.0': 1, + 'a.1': 2, + 'a.2': 3, + 'a.3.b': 4, + }, + ) + }) +}) // since we're not resetting the db every time we need to change this -let count = 1; -const makeTopic = () => `topic-${count++}`; +let count = 1 +const makeTopic = () => `topic-${count++}` -describe("getFilteredSubs", () => { +describe('getFilteredSubs', () => { before(async () => { - await tables.start({ cwd: "./mocks/arc-basic-events", quiet: true }); - }); + await tables.start({ cwd: './mocks/arc-basic-events', quiet: true }) + }) after(async () => { - await tables.end(); - }); + await tables.end() + }) - it("can match on no filter", async () => { - const topic = makeTopic(); - const server = await mockServerContext(); + it('can match on no filter', async () => { + const topic = makeTopic() + const server = await mockServerContext() const subscription = { - id: "1", + id: '1', topic, filter: {}, - subscriptionId: "1", + subscriptionId: '1', subscription: {} as any, - connectionId: "abcd", + connectionId: 'abcd', connectionInitPayload: {}, requestContext: {} as any, ttl: Math.floor(Date.now() / 1000) + 100000, createdAt: Date.now(), - }; + } - await server.models.subscription.put(subscription); + await server.models.subscription.put(subscription) assert.containSubset( await getFilteredSubs({ server, - event: { topic, payload: { language: "en" } }, + event: { topic, payload: { language: 'en' } }, }), - [{ topic, id: "1" }] - ); - }); + [{ topic, id: '1' }], + ) + }) - it("can match on payload", async () => { - const topic = makeTopic(); - const server = await mockServerContext(); + it('can match on payload', async () => { + const topic = makeTopic() + const server = await mockServerContext() const subscription = { - id: "2", + id: '2', topic, - filter: { language: "en" }, - subscriptionId: "2", + filter: { language: 'en' }, + subscriptionId: '2', subscription: {} as any, - connectionId: "abcd", + connectionId: 'abcd', connectionInitPayload: {}, requestContext: {} as any, ttl: Math.floor(Date.now() / 1000) + 100000, createdAt: Date.now(), - }; + } - await server.models.subscription.put(subscription); + await server.models.subscription.put(subscription) assert.containSubset( await getFilteredSubs({ server, - event: { topic, payload: { language: "en" } }, + event: { topic, payload: { language: 'en' } }, }), - [{ topic, id: "2" }] - ); + [{ topic, id: '2' }], + ) assert.deepEqual( await getFilteredSubs({ server, - event: { topic, payload: { language: "en-gb" } }, + event: { topic, payload: { language: 'en-gb' } }, }), - [] - ); - }); + [], + ) + }) - it("can match on nested payload", async () => { - const topic = makeTopic(); - const server = await mockServerContext(); + it('can match on nested payload', async () => { + const topic = makeTopic() + const server = await mockServerContext() const subscription = { - id: "2", + id: '2', topic, - filter: { meta: { user: "foo" }, message: { content: "hi" } }, - subscriptionId: "2", + filter: { meta: { user: 'foo' }, message: { content: 'hi' } }, + subscriptionId: '2', subscription: {} as any, - connectionId: "abcd", + connectionId: 'abcd', connectionInitPayload: {}, requestContext: {} as any, ttl: Math.floor(Date.now() / 1000) + 100000, createdAt: Date.now(), - }; + } - await server.models.subscription.put(subscription); + await server.models.subscription.put(subscription) assert.containSubset( await getFilteredSubs({ server, event: { topic, - payload: { meta: { user: "foo" }, message: { content: "hi" } }, + payload: { meta: { user: 'foo' }, message: { content: 'hi' } }, }, }), - [{ topic, id: "2" }] - ); + [{ topic, id: '2' }], + ) assert.deepEqual( await getFilteredSubs({ server, event: { topic, - payload: { meta: { user: "lol" }, message: { content: "bye" } }, + payload: { meta: { user: 'lol' }, message: { content: 'bye' } }, }, }), - [] - ); - }); + [], + ) + }) - it("can match on no payload", async () => { - const topic = makeTopic(); - const server = await mockServerContext(); + it('can match on no payload', async () => { + const topic = makeTopic() + const server = await mockServerContext() const subscription = { - id: "1234567", + id: '1234567', topic, - filter: { language: "en " }, - subscriptionId: "12345", + filter: { language: 'en ' }, + subscriptionId: '12345', subscription: {} as any, - connectionId: "1234", + connectionId: '1234', connectionInitPayload: {}, requestContext: {} as any, ttl: Math.floor(Date.now() / 1000) + 100000, createdAt: Date.now(), - }; + } - await server.models.subscription.put(subscription); - const subscriptions = await getFilteredSubs({ server, event: { topic } }); - assert.containSubset(subscriptions, [{ topic, id: "1234567" }]); - }); - it("can match on connectionId"); - it("can match on topic key"); -}); + await server.models.subscription.put(subscription) + const subscriptions = await getFilteredSubs({ server, event: { topic } }) + assert.containSubset(subscriptions, [{ topic, id: '1234567' }]) + }) + it('can match on connectionId') + it('can match on topic key') +}) diff --git a/lib/pubsub/getFilteredSubs.ts b/lib/pubsub/getFilteredSubs.ts index 558745b3..7e701392 100644 --- a/lib/pubsub/getFilteredSubs.ts +++ b/lib/pubsub/getFilteredSubs.ts @@ -1,102 +1,102 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ -import { collect } from "streaming-iterables"; -import { ServerClosure, Subscription } from "../types"; +import { collect } from 'streaming-iterables' +import { ServerClosure, Subscription } from '../types' export const getFilteredSubs = async ({ server, event, }: { - server: Omit; - event: { topic: string; payload?: Record }; + server: Omit + event: { topic: string, payload?: Record } }): Promise => { if (!event.payload || Object.keys(event.payload).length === 0) { - server.log("getFilteredSubs", { event }); + server.log('getFilteredSubs', { event }) const iterator = server.models.subscription.query({ - IndexName: "TopicIndex", - ExpressionAttributeNames: { "#a": "topic" }, - ExpressionAttributeValues: { ":1": event.topic }, - KeyConditionExpression: "#a = :1", - }); + IndexName: 'TopicIndex', + ExpressionAttributeNames: { '#a': 'topic' }, + ExpressionAttributeValues: { ':1': event.topic }, + KeyConditionExpression: '#a = :1', + }) - return await collect(iterator); + return await collect(iterator) } - const flattenPayload = collapseKeys(event.payload); + const flattenPayload = collapseKeys(event.payload) - const filterExpressions: string[] = []; + const filterExpressions: string[] = [] const expressionAttributeValues: { - [key: string]: string | number | boolean; - } = {}; - const expressionAttributeNames: { [key: string]: string } = {}; + [key: string]: string | number | boolean + } = {} + const expressionAttributeNames: { [key: string]: string } = {} - let attributeCounter = 0; + let attributeCounter = 0 for (const [key, value] of Object.entries(flattenPayload)) { - let aliasNumber = attributeCounter++; - const keyParts = key.split("."); + const aliasNumber = attributeCounter++ + const keyParts = key.split('.') const keyPartsAttributeName = keyParts .map((part, index) => `#${aliasNumber + index}`) - .join("."); - key.split(".").forEach((keyPart, index) => { - expressionAttributeNames[`#${aliasNumber + index}`] = keyPart; - attributeCounter += index; - }); - expressionAttributeValues[`:${aliasNumber}`] = value; + .join('.') + key.split('.').forEach((keyPart, index) => { + expressionAttributeNames[`#${aliasNumber + index}`] = keyPart + attributeCounter += index + }) + expressionAttributeValues[`:${aliasNumber}`] = value filterExpressions.push( - `(#filter.${keyPartsAttributeName} = :${aliasNumber} OR attribute_not_exists(#filter.${keyPartsAttributeName}))` - ); + `(#filter.${keyPartsAttributeName} = :${aliasNumber} OR attribute_not_exists(#filter.${keyPartsAttributeName}))`, + ) } - server.log("getFilteredSubs", { + server.log('getFilteredSubs', { event, expressionAttributeNames, expressionAttributeValues, filterExpressions, - }); + }) const iterator = server.models.subscription.query({ - IndexName: "TopicIndex", + IndexName: 'TopicIndex', ExpressionAttributeNames: { - "#hashKey": "topic", - "#filter": "filter", + '#hashKey': 'topic', + '#filter': 'filter', ...expressionAttributeNames, }, ExpressionAttributeValues: { - ":hashKey": event.topic, + ':hashKey': event.topic, ...expressionAttributeValues, }, - KeyConditionExpression: "#hashKey = :hashKey", - FilterExpression: filterExpressions.join(" AND ") || undefined, - }); + KeyConditionExpression: '#hashKey = :hashKey', + FilterExpression: filterExpressions.join(' AND ') || undefined, + }) - return await collect(iterator); -}; + return await collect(iterator) +} export const collapseKeys = ( - obj: Record + obj: Record, ): Record => { - const record = {}; + const record = {} for (const [k1, v1] of Object.entries(obj)) { if ( - typeof v1 === "string" || - typeof v1 === "number" || - typeof v1 === "boolean" + typeof v1 === 'string' || + typeof v1 === 'number' || + typeof v1 === 'boolean' ) { - record[k1] = v1; - continue; + record[k1] = v1 + continue } - if (v1 && typeof v1 === "object") { - const next = {}; + if (v1 && typeof v1 === 'object') { + const next = {} for (const [k2, v2] of Object.entries(v1)) { - next[`${k1}.${k2}`] = v2; + next[`${k1}.${k2}`] = v2 } for (const [k1, v1] of Object.entries(collapseKeys(next))) { - record[k1] = v1; + record[k1] = v1 } } } - return record; -}; + return record +}