From 440e1769c1e6f4642500a40ea0929439d0ba6bec Mon Sep 17 00:00:00 2001 From: Derrick Hammer Date: Mon, 9 Sep 2024 03:44:52 -0400 Subject: [PATCH 1/7] feat: add support for custom headers in trustless gateway Added an optional `headers` property to the `TrustlessGatewayBlockBroker` and `TrustlessGatewaySession` options, allowing users to specify custom headers for HTTP requests. Changes include: * Added `headers` property to `TrustlessGatewayBlockBroker` and `TrustlessGatewaySession` options * Updated `findHttpGatewayProviders` function to accept an optional `headers` object * Passed `headers` object to `TrustlessGateway` constructor * Updated `TrustlessGateway` constructor to accept an optional `headers` object * Added `headers` to HTTP request options in `TrustlessGateway` --- .../block-brokers/src/trustless-gateway/broker.ts | 12 ++++++++++-- .../block-brokers/src/trustless-gateway/index.ts | 2 ++ .../block-brokers/src/trustless-gateway/session.ts | 6 ++++-- .../src/trustless-gateway/trustless-gateway.ts | 7 +++++-- .../block-brokers/src/trustless-gateway/utils.ts | 4 ++-- .../block-brokers/test/trustless-gateway.spec.ts | 2 +- 6 files changed, 24 insertions(+), 9 deletions(-) diff --git a/packages/block-brokers/src/trustless-gateway/broker.ts b/packages/block-brokers/src/trustless-gateway/broker.ts index 05d2b518a..e8ca65eeb 100644 --- a/packages/block-brokers/src/trustless-gateway/broker.ts +++ b/packages/block-brokers/src/trustless-gateway/broker.ts @@ -22,6 +22,8 @@ export interface CreateTrustlessGatewaySessionOptions extends CreateSessionOptio * @default false */ allowLocal?: boolean + + headers?: Record } /** @@ -31,6 +33,7 @@ export interface CreateTrustlessGatewaySessionOptions extends CreateSessionOptio export class TrustlessGatewayBlockBroker implements BlockBroker { private readonly allowInsecure: boolean private readonly allowLocal: boolean + private readonly headers?: Record private readonly routing: Routing private readonly log: Logger private readonly logger: ComponentLogger @@ -41,12 +44,13 @@ export class TrustlessGatewayBlockBroker implements BlockBroker = {}): Promise { const aggregateErrors: Error[] = [] - for await (const gateway of findHttpGatewayProviders(cid, this.routing, this.logger, this.allowInsecure, this.allowLocal, options)) { + for await (const gateway of findHttpGatewayProviders(cid, this.routing, this.logger, this.allowInsecure, this.allowLocal, this.headers, options)) { this.log('getting block for %c from %s', cid, gateway.url) try { @@ -93,7 +97,11 @@ export class TrustlessGatewayBlockBroker implements BlockBroker } export interface TrustlessGatewayBlockBrokerComponents { diff --git a/packages/block-brokers/src/trustless-gateway/session.ts b/packages/block-brokers/src/trustless-gateway/session.ts index 9d26d7435..c4db733c3 100644 --- a/packages/block-brokers/src/trustless-gateway/session.ts +++ b/packages/block-brokers/src/trustless-gateway/session.ts @@ -18,6 +18,7 @@ class TrustlessGatewaySession extends AbstractSession constructor (components: TrustlessGatewaySessionComponents, init: CreateTrustlessGatewaySessionOptions) { super(components, { @@ -28,6 +29,7 @@ class TrustlessGatewaySession extends AbstractSession { @@ -41,8 +43,8 @@ class TrustlessGatewaySession extends AbstractSession { - yield * findHttpGatewayProviders(cid, this.routing, this.logger, this.allowInsecure, this.allowLocal, options) + async * findNewProviders (cid: CID, options: AbortOptions = {}, headers: Record = {}): AsyncGenerator { + yield * findHttpGatewayProviders(cid, this.routing, this.logger, this.allowInsecure, this.allowLocal, this.headers, options) } toEvictionKey (provider: TrustlessGateway): Uint8Array | string { diff --git a/packages/block-brokers/src/trustless-gateway/trustless-gateway.ts b/packages/block-brokers/src/trustless-gateway/trustless-gateway.ts index 97a5faef5..88324eeec 100644 --- a/packages/block-brokers/src/trustless-gateway/trustless-gateway.ts +++ b/packages/block-brokers/src/trustless-gateway/trustless-gateway.ts @@ -54,9 +54,11 @@ export class TrustlessGateway { readonly #pendingResponses = new Map>() private readonly log: Logger + private readonly headers: Record - constructor (url: URL | string, logger: ComponentLogger) { + constructor (url: URL | string, headers: Record, logger: ComponentLogger) { this.url = url instanceof URL ? url : new URL(url) + this.headers = headers this.log = logger.forComponent(`helia:trustless-gateway-block-broker:${this.url.hostname}`) } @@ -106,7 +108,8 @@ export class TrustlessGateway { pendingResponse = fetch(gwUrl.toString(), { signal: innerController.signal, headers: { - Accept: 'application/vnd.ipld.raw' + Accept: 'application/vnd.ipld.raw', + ...this.headers }, cache: 'force-cache' }).then(async (res) => { diff --git a/packages/block-brokers/src/trustless-gateway/utils.ts b/packages/block-brokers/src/trustless-gateway/utils.ts index 820c44ff1..376eb42cf 100644 --- a/packages/block-brokers/src/trustless-gateway/utils.ts +++ b/packages/block-brokers/src/trustless-gateway/utils.ts @@ -33,7 +33,7 @@ export function filterNonHTTPMultiaddrs (multiaddrs: Multiaddr[], allowInsecure: }) } -export async function * findHttpGatewayProviders (cid: CID, routing: Routing, logger: ComponentLogger, allowInsecure: boolean, allowLocal: boolean, options?: AbortOptions): AsyncGenerator { +export async function * findHttpGatewayProviders (cid: CID, routing: Routing, logger: ComponentLogger, allowInsecure: boolean, allowLocal: boolean, headers: Record = {}, options: AbortOptions = {}): AsyncGenerator { for await (const provider of routing.findProviders(cid, options)) { // require http(s) addresses const httpAddresses = filterNonHTTPMultiaddrs(provider.multiaddrs, allowInsecure, allowLocal) @@ -48,6 +48,6 @@ export async function * findHttpGatewayProviders (cid: CID, routing: Routing, lo // etc const uri = multiaddrToUri(httpAddresses[0]) - yield new TrustlessGateway(uri, logger) + yield new TrustlessGateway(uri, headers, logger) } } diff --git a/packages/block-brokers/test/trustless-gateway.spec.ts b/packages/block-brokers/test/trustless-gateway.spec.ts index 8b9496632..a135d070b 100644 --- a/packages/block-brokers/test/trustless-gateway.spec.ts +++ b/packages/block-brokers/test/trustless-gateway.spec.ts @@ -149,7 +149,7 @@ describe('trustless-gateway-block-broker', () => { if (process.env.TRUSTLESS_GATEWAY == null) { return this.skip() } - const trustlessGateway = new TrustlessGateway(process.env.TRUSTLESS_GATEWAY, defaultLogger()) + const trustlessGateway = new TrustlessGateway(process.env.TRUSTLESS_GATEWAY, {}, defaultLogger()) // Call getRawBlock multiple times with the same CID const promises = Array.from({ length: 10 }, async () => trustlessGateway.getRawBlock(cid)) From fbf8e68defb376e7c9ea6a4c81feb4034ec32b06 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Thu, 27 Feb 2025 14:53:57 -0600 Subject: [PATCH 2/7] chore: allow test trustless gateway server introspection --- packages/block-brokers/.aegir.js | 39 ++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/packages/block-brokers/.aegir.js b/packages/block-brokers/.aegir.js index 161be0a71..db259de35 100644 --- a/packages/block-brokers/.aegir.js +++ b/packages/block-brokers/.aegir.js @@ -1,6 +1,23 @@ import cors from 'cors' import polka from 'polka' + +/** + * Middleware to log requests + */ +const requestLogs = [] +let enableLogs = false +function logRequests(req, res, next) { + if (!req.url.includes('/logs') && enableLogs) { + requestLogs.push({ + method: req.method, + url: req.url, + headers: req.headers + }) + } + next() +} + /** @type {import('aegir').PartialOptions} */ const options = { test: { @@ -10,6 +27,7 @@ const options = { host: '127.0.0.1' }) goodGateway.use(cors()) + goodGateway.use(logRequests) goodGateway.all('/ipfs/bafkreiefnkxuhnq3536qo2i2w3tazvifek4mbbzb6zlq3ouhprjce5c3aq', (req, res) => { res.writeHead(200, { 'content-type': 'application/octet-stream', @@ -29,7 +47,28 @@ const options = { res.end(Uint8Array.from([104, 101, 108, 108, 111])) }) + goodGateway.all('/logs', (req, res) => { + res.writeHead(200, { 'content-type': 'application/json' }) + res.end(JSON.stringify(requestLogs)) + }) + goodGateway.all('/logs/enable', (req, res) => { + enableLogs = true + res.writeHead(200, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ message: 'Logging enabled' })) + }) + goodGateway.all('/logs/disable', (req, res) => { + enableLogs = false + res.writeHead(200, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ message: 'Logging disabled' })) + }) + goodGateway.all('/logs/clear', (req, res) => { + requestLogs.length = 0 + res.writeHead(200, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ message: 'Logs cleared' })) + }) + await goodGateway.listen() + const { port: goodGatewayPort } = goodGateway.server.address() const badGateway = polka({ From 73545e4c1cfea74e0c291e0c265523b026b657bc Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Thu, 27 Feb 2025 14:56:11 -0600 Subject: [PATCH 3/7] feat: allow modification of fetch options to trustless gateway --- .../src/trustless-gateway/broker.ts | 14 ++++---- .../src/trustless-gateway/index.ts | 3 +- .../src/trustless-gateway/session.ts | 8 ++--- .../trustless-gateway/trustless-gateway.ts | 26 +++++++++++---- .../src/trustless-gateway/utils.ts | 6 ++-- .../test/trustless-gateway.spec.ts | 32 ++++++++++++++++++- 6 files changed, 65 insertions(+), 24 deletions(-) diff --git a/packages/block-brokers/src/trustless-gateway/broker.ts b/packages/block-brokers/src/trustless-gateway/broker.ts index e8ca65eeb..b484d464f 100644 --- a/packages/block-brokers/src/trustless-gateway/broker.ts +++ b/packages/block-brokers/src/trustless-gateway/broker.ts @@ -2,6 +2,7 @@ import { createTrustlessGatewaySession } from './session.js' import { findHttpGatewayProviders } from './utils.js' import { DEFAULT_ALLOW_INSECURE, DEFAULT_ALLOW_LOCAL } from './index.js' import type { TrustlessGatewayBlockBrokerInit, TrustlessGatewayBlockBrokerComponents, TrustlessGatewayGetBlockProgressEvents } from './index.js' +import type { TransformRequestInit } from './trustless-gateway.js' import type { Routing, BlockRetrievalOptions, BlockBroker, CreateSessionOptions } from '@helia/interface' import type { ComponentLogger, Logger } from '@libp2p/interface' import type { CID } from 'multiformats/cid' @@ -23,7 +24,7 @@ export interface CreateTrustlessGatewaySessionOptions extends CreateSessionOptio */ allowLocal?: boolean - headers?: Record + transformRequestInit?: TransformRequestInit } /** @@ -33,7 +34,7 @@ export interface CreateTrustlessGatewaySessionOptions extends CreateSessionOptio export class TrustlessGatewayBlockBroker implements BlockBroker { private readonly allowInsecure: boolean private readonly allowLocal: boolean - private readonly headers?: Record + private readonly transformRequestInit?: TransformRequestInit private readonly routing: Routing private readonly log: Logger private readonly logger: ComponentLogger @@ -44,13 +45,13 @@ export class TrustlessGatewayBlockBroker implements BlockBroker = {}): Promise { const aggregateErrors: Error[] = [] - for await (const gateway of findHttpGatewayProviders(cid, this.routing, this.logger, this.allowInsecure, this.allowLocal, this.headers, options)) { + for await (const gateway of findHttpGatewayProviders(cid, this.routing, this.logger, this.allowInsecure, this.allowLocal, this.transformRequestInit, options)) { this.log('getting block for %c from %s', cid, gateway.url) try { @@ -98,10 +99,7 @@ export class TrustlessGatewayBlockBroker implements BlockBroker + transformRequestInit?: TransformRequestInit } export interface TrustlessGatewayBlockBrokerComponents { diff --git a/packages/block-brokers/src/trustless-gateway/session.ts b/packages/block-brokers/src/trustless-gateway/session.ts index c4db733c3..f2ea4e4db 100644 --- a/packages/block-brokers/src/trustless-gateway/session.ts +++ b/packages/block-brokers/src/trustless-gateway/session.ts @@ -3,7 +3,7 @@ import { findHttpGatewayProviders } from './utils.js' import { DEFAULT_ALLOW_INSECURE, DEFAULT_ALLOW_LOCAL } from './index.js' import type { CreateTrustlessGatewaySessionOptions } from './broker.js' import type { TrustlessGatewayGetBlockProgressEvents } from './index.js' -import type { TrustlessGateway } from './trustless-gateway.js' +import type { TransformRequestInit, TrustlessGateway } from './trustless-gateway.js' import type { BlockRetrievalOptions, Routing } from '@helia/interface' import type { ComponentLogger } from '@libp2p/interface' import type { AbortOptions } from 'interface-store' @@ -18,7 +18,7 @@ class TrustlessGatewaySession extends AbstractSession + private readonly transformRequestInit?: TransformRequestInit constructor (components: TrustlessGatewaySessionComponents, init: CreateTrustlessGatewaySessionOptions) { super(components, { @@ -29,7 +29,7 @@ class TrustlessGatewaySession extends AbstractSession { @@ -44,7 +44,7 @@ class TrustlessGatewaySession extends AbstractSession = {}): AsyncGenerator { - yield * findHttpGatewayProviders(cid, this.routing, this.logger, this.allowInsecure, this.allowLocal, this.headers, options) + yield * findHttpGatewayProviders(cid, this.routing, this.logger, this.allowInsecure, this.allowLocal, this.transformRequestInit, options) } toEvictionKey (provider: TrustlessGateway): Uint8Array | string { diff --git a/packages/block-brokers/src/trustless-gateway/trustless-gateway.ts b/packages/block-brokers/src/trustless-gateway/trustless-gateway.ts index 88324eeec..db6afac07 100644 --- a/packages/block-brokers/src/trustless-gateway/trustless-gateway.ts +++ b/packages/block-brokers/src/trustless-gateway/trustless-gateway.ts @@ -10,6 +10,15 @@ export interface TrustlessGatewayStats { pendingResponses?: number } +export interface TransformRequestInit { + (defaultReqInit: RequestInit): Promise | RequestInit +} + +export interface TrustlessGatewayComponents { + logger: ComponentLogger + transformRequestInit?: TransformRequestInit +} + /** * A `TrustlessGateway` keeps track of the number of attempts, errors, and * successes for a given gateway url so that we can prioritize gateways that @@ -54,11 +63,11 @@ export class TrustlessGateway { readonly #pendingResponses = new Map>() private readonly log: Logger - private readonly headers: Record + private readonly transformRequestInit?: TransformRequestInit - constructor (url: URL | string, headers: Record, logger: ComponentLogger) { + constructor (url: URL | string, { logger, transformRequestInit }: TrustlessGatewayComponents) { this.url = url instanceof URL ? url : new URL(url) - this.headers = headers + this.transformRequestInit = transformRequestInit this.log = logger.forComponent(`helia:trustless-gateway-block-broker:${this.url.hostname}`) } @@ -105,14 +114,17 @@ export class TrustlessGateway { let pendingResponse: Promise | undefined = this.#pendingResponses.get(blockId) if (pendingResponse == null) { this.#attempts++ - pendingResponse = fetch(gwUrl.toString(), { + const defaultReqInit: RequestInit = { signal: innerController.signal, headers: { - Accept: 'application/vnd.ipld.raw', - ...this.headers + Accept: 'application/vnd.ipld.raw' }, cache: 'force-cache' - }).then(async (res) => { + } + + const reqInit: RequestInit = this.transformRequestInit != null ? await this.transformRequestInit(defaultReqInit) : defaultReqInit + + pendingResponse = fetch(gwUrl.toString(), reqInit).then(async (res) => { this.log('GET %s %d', gwUrl, res.status) if (!res.ok) { this.#errors++ diff --git a/packages/block-brokers/src/trustless-gateway/utils.ts b/packages/block-brokers/src/trustless-gateway/utils.ts index 376eb42cf..4e12b9f2b 100644 --- a/packages/block-brokers/src/trustless-gateway/utils.ts +++ b/packages/block-brokers/src/trustless-gateway/utils.ts @@ -1,7 +1,7 @@ import { isPrivateIp } from '@libp2p/utils/private-ip' import { DNS, HTTP, HTTPS } from '@multiformats/multiaddr-matcher' import { multiaddrToUri } from '@multiformats/multiaddr-to-uri' -import { TrustlessGateway } from './trustless-gateway.js' +import { TrustlessGateway, type TransformRequestInit } from './trustless-gateway.js' import type { Routing } from '@helia/interface' import type { ComponentLogger } from '@libp2p/interface' import type { AbortOptions, Multiaddr } from '@multiformats/multiaddr' @@ -33,7 +33,7 @@ export function filterNonHTTPMultiaddrs (multiaddrs: Multiaddr[], allowInsecure: }) } -export async function * findHttpGatewayProviders (cid: CID, routing: Routing, logger: ComponentLogger, allowInsecure: boolean, allowLocal: boolean, headers: Record = {}, options: AbortOptions = {}): AsyncGenerator { +export async function * findHttpGatewayProviders (cid: CID, routing: Routing, logger: ComponentLogger, allowInsecure: boolean, allowLocal: boolean, transformRequestInit?: TransformRequestInit, options: AbortOptions = {}): AsyncGenerator { for await (const provider of routing.findProviders(cid, options)) { // require http(s) addresses const httpAddresses = filterNonHTTPMultiaddrs(provider.multiaddrs, allowInsecure, allowLocal) @@ -48,6 +48,6 @@ export async function * findHttpGatewayProviders (cid: CID, routing: Routing, lo // etc const uri = multiaddrToUri(httpAddresses[0]) - yield new TrustlessGateway(uri, headers, logger) + yield new TrustlessGateway(uri, { logger, transformRequestInit }) } } diff --git a/packages/block-brokers/test/trustless-gateway.spec.ts b/packages/block-brokers/test/trustless-gateway.spec.ts index a135d070b..bfafeb1b2 100644 --- a/packages/block-brokers/test/trustless-gateway.spec.ts +++ b/packages/block-brokers/test/trustless-gateway.spec.ts @@ -149,7 +149,7 @@ describe('trustless-gateway-block-broker', () => { if (process.env.TRUSTLESS_GATEWAY == null) { return this.skip() } - const trustlessGateway = new TrustlessGateway(process.env.TRUSTLESS_GATEWAY, {}, defaultLogger()) + const trustlessGateway = new TrustlessGateway(process.env.TRUSTLESS_GATEWAY, { logger: defaultLogger() }) // Call getRawBlock multiple times with the same CID const promises = Array.from({ length: 10 }, async () => trustlessGateway.getRawBlock(cid)) @@ -171,4 +171,34 @@ describe('trustless-gateway-block-broker', () => { pendingResponses: 0 // the queue is empty }) }) + + it('can pass custom headers to the gateway', async function () { + if (process.env.TRUSTLESS_GATEWAY == null) { + return this.skip() + } + const cid = CID.parse('bafkreiefnkxuhnq3536qo2i2w3tazvifek4mbbzb6zlq3ouhprjce5c3aq') + + const trustlessGateway = new TrustlessGateway(process.env.TRUSTLESS_GATEWAY, { + logger: defaultLogger(), + transformRequestInit: (requestInit) => { + requestInit.headers = { + ...requestInit.headers, + 'X-My-Header': 'my-value' + } + + return requestInit + } + }) + + await fetch(`${process.env.TRUSTLESS_GATEWAY}/logs/enable`) + await trustlessGateway.getRawBlock(cid) + await fetch(`${process.env.TRUSTLESS_GATEWAY}/logs/disable`) + + const reqLogs = await fetch(`${process.env.TRUSTLESS_GATEWAY}/logs`) + const logs = await reqLogs.json() + + // assert that fetch was called with the custom header + expect(logs).to.have.lengthOf(1) + expect(logs[0].headers['x-my-header']).to.equal('my-value') + }) }) From 8f925807b9fa4b6cf35dd92b31beb73a910e6bc5 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Thu, 27 Feb 2025 13:03:57 -0800 Subject: [PATCH 4/7] chore: apply self suggestions from code review --- packages/block-brokers/.aegir.js | 1 - packages/block-brokers/src/trustless-gateway/broker.ts | 4 +++- packages/block-brokers/src/trustless-gateway/index.ts | 4 +++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/block-brokers/.aegir.js b/packages/block-brokers/.aegir.js index db259de35..69005da3b 100644 --- a/packages/block-brokers/.aegir.js +++ b/packages/block-brokers/.aegir.js @@ -68,7 +68,6 @@ const options = { }) await goodGateway.listen() - const { port: goodGatewayPort } = goodGateway.server.address() const badGateway = polka({ diff --git a/packages/block-brokers/src/trustless-gateway/broker.ts b/packages/block-brokers/src/trustless-gateway/broker.ts index b484d464f..e0020407d 100644 --- a/packages/block-brokers/src/trustless-gateway/broker.ts +++ b/packages/block-brokers/src/trustless-gateway/broker.ts @@ -23,7 +23,9 @@ export interface CreateTrustlessGatewaySessionOptions extends CreateSessionOptio * @default false */ allowLocal?: boolean - + /** + * Provide a function that will be called before querying trustless-gateways. This lets you modify the fetch options to pass custom headers or other necessary things. + */ transformRequestInit?: TransformRequestInit } diff --git a/packages/block-brokers/src/trustless-gateway/index.ts b/packages/block-brokers/src/trustless-gateway/index.ts index f035a5b26..9aab42bae 100644 --- a/packages/block-brokers/src/trustless-gateway/index.ts +++ b/packages/block-brokers/src/trustless-gateway/index.ts @@ -26,7 +26,9 @@ export interface TrustlessGatewayBlockBrokerInit { * @default false */ allowLocal?: boolean - + /** + * Provide a function that will be called before querying trustless-gateways. This lets you modify the fetch options to pass custom headers or other necessary things. + */ transformRequestInit?: TransformRequestInit } From cd1f716593624ae986089dfcd94f3add513d850e Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Fri, 28 Feb 2025 09:43:10 -0600 Subject: [PATCH 5/7] chore: new header test passes --- packages/block-brokers/.aegir.js | 5 +++++ packages/block-brokers/test/trustless-gateway.spec.ts | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/block-brokers/.aegir.js b/packages/block-brokers/.aegir.js index 69005da3b..93c6b9f68 100644 --- a/packages/block-brokers/.aegir.js +++ b/packages/block-brokers/.aegir.js @@ -46,6 +46,11 @@ const options = { // "hello" res.end(Uint8Array.from([104, 101, 108, 108, 111])) }) + goodGateway.all('/ipfs/*', (req, res) => { + // succeeds with empty block for any other CID + res.writeHead(200) + res.end(Uint8Array.from([])) + }) goodGateway.all('/logs', (req, res) => { res.writeHead(200, { 'content-type': 'application/json' }) diff --git a/packages/block-brokers/test/trustless-gateway.spec.ts b/packages/block-brokers/test/trustless-gateway.spec.ts index bfafeb1b2..01ea202cb 100644 --- a/packages/block-brokers/test/trustless-gateway.spec.ts +++ b/packages/block-brokers/test/trustless-gateway.spec.ts @@ -176,7 +176,7 @@ describe('trustless-gateway-block-broker', () => { if (process.env.TRUSTLESS_GATEWAY == null) { return this.skip() } - const cid = CID.parse('bafkreiefnkxuhnq3536qo2i2w3tazvifek4mbbzb6zlq3ouhprjce5c3aq') + const cid = CID.parse('bafybeic3q4y65yxu3yckr76q63bcvanhklwf6cwxuacnrot6v3gykrgsvq') const trustlessGateway = new TrustlessGateway(process.env.TRUSTLESS_GATEWAY, { logger: defaultLogger(), From 141fd93e3ec261b4f6c23b2f34a3d27f52e6997b Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Mon, 10 Mar 2025 14:43:47 -0500 Subject: [PATCH 6/7] test: fix headers test and remove unused arg --- packages/block-brokers/src/trustless-gateway/session.ts | 2 +- packages/block-brokers/test/trustless-gateway.spec.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/block-brokers/src/trustless-gateway/session.ts b/packages/block-brokers/src/trustless-gateway/session.ts index f2ea4e4db..a67f558fd 100644 --- a/packages/block-brokers/src/trustless-gateway/session.ts +++ b/packages/block-brokers/src/trustless-gateway/session.ts @@ -43,7 +43,7 @@ class TrustlessGatewaySession extends AbstractSession = {}): AsyncGenerator { + async * findNewProviders (cid: CID, options: AbortOptions = {}): AsyncGenerator { yield * findHttpGatewayProviders(cid, this.routing, this.logger, this.allowInsecure, this.allowLocal, this.transformRequestInit, options) } diff --git a/packages/block-brokers/test/trustless-gateway.spec.ts b/packages/block-brokers/test/trustless-gateway.spec.ts index 01ea202cb..8a43c683c 100644 --- a/packages/block-brokers/test/trustless-gateway.spec.ts +++ b/packages/block-brokers/test/trustless-gateway.spec.ts @@ -196,6 +196,7 @@ describe('trustless-gateway-block-broker', () => { const reqLogs = await fetch(`${process.env.TRUSTLESS_GATEWAY}/logs`) const logs = await reqLogs.json() + await fetch(`${process.env.TRUSTLESS_GATEWAY}/logs/clear`) // assert that fetch was called with the custom header expect(logs).to.have.lengthOf(1) From f0bb569abfaa2a1bf8d600cb4f3875a5f831c762 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Mon, 10 Mar 2025 15:44:37 -0500 Subject: [PATCH 7/7] chore: add transformRequestInit to options object --- packages/block-brokers/src/trustless-gateway/broker.ts | 2 +- packages/block-brokers/src/trustless-gateway/session.ts | 2 +- packages/block-brokers/src/trustless-gateway/utils.ts | 8 ++++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/block-brokers/src/trustless-gateway/broker.ts b/packages/block-brokers/src/trustless-gateway/broker.ts index e0020407d..db08691c2 100644 --- a/packages/block-brokers/src/trustless-gateway/broker.ts +++ b/packages/block-brokers/src/trustless-gateway/broker.ts @@ -53,7 +53,7 @@ export class TrustlessGatewayBlockBroker implements BlockBroker = {}): Promise { const aggregateErrors: Error[] = [] - for await (const gateway of findHttpGatewayProviders(cid, this.routing, this.logger, this.allowInsecure, this.allowLocal, this.transformRequestInit, options)) { + for await (const gateway of findHttpGatewayProviders(cid, this.routing, this.logger, this.allowInsecure, this.allowLocal, { ...options, transformRequestInit: this.transformRequestInit })) { this.log('getting block for %c from %s', cid, gateway.url) try { diff --git a/packages/block-brokers/src/trustless-gateway/session.ts b/packages/block-brokers/src/trustless-gateway/session.ts index a67f558fd..711d980ae 100644 --- a/packages/block-brokers/src/trustless-gateway/session.ts +++ b/packages/block-brokers/src/trustless-gateway/session.ts @@ -44,7 +44,7 @@ class TrustlessGatewaySession extends AbstractSession { - yield * findHttpGatewayProviders(cid, this.routing, this.logger, this.allowInsecure, this.allowLocal, this.transformRequestInit, options) + yield * findHttpGatewayProviders(cid, this.routing, this.logger, this.allowInsecure, this.allowLocal, { ...options, transformRequestInit: this.transformRequestInit }) } toEvictionKey (provider: TrustlessGateway): Uint8Array | string { diff --git a/packages/block-brokers/src/trustless-gateway/utils.ts b/packages/block-brokers/src/trustless-gateway/utils.ts index 4e12b9f2b..6924ede1f 100644 --- a/packages/block-brokers/src/trustless-gateway/utils.ts +++ b/packages/block-brokers/src/trustless-gateway/utils.ts @@ -33,7 +33,11 @@ export function filterNonHTTPMultiaddrs (multiaddrs: Multiaddr[], allowInsecure: }) } -export async function * findHttpGatewayProviders (cid: CID, routing: Routing, logger: ComponentLogger, allowInsecure: boolean, allowLocal: boolean, transformRequestInit?: TransformRequestInit, options: AbortOptions = {}): AsyncGenerator { +export interface FindHttpGatewayProvidersOptions extends AbortOptions { + transformRequestInit?: TransformRequestInit +} + +export async function * findHttpGatewayProviders (cid: CID, routing: Routing, logger: ComponentLogger, allowInsecure: boolean, allowLocal: boolean, options: FindHttpGatewayProvidersOptions = {}): AsyncGenerator { for await (const provider of routing.findProviders(cid, options)) { // require http(s) addresses const httpAddresses = filterNonHTTPMultiaddrs(provider.multiaddrs, allowInsecure, allowLocal) @@ -48,6 +52,6 @@ export async function * findHttpGatewayProviders (cid: CID, routing: Routing, lo // etc const uri = multiaddrToUri(httpAddresses[0]) - yield new TrustlessGateway(uri, { logger, transformRequestInit }) + yield new TrustlessGateway(uri, { logger, transformRequestInit: options.transformRequestInit }) } }