From ac9e617da48f562b8de963ceb6f0a924432980e0 Mon Sep 17 00:00:00 2001 From: araddcc002 Date: Wed, 11 Sep 2024 15:58:22 +0000 Subject: [PATCH 01/32] wip for file scanning connector --- backend/config/default.cjs | 6 +- backend/src/connectors/fileScanning/Base.ts | 11 +++ backend/src/connectors/fileScanning/clamAv.ts | 75 +++++++++++++++++++ backend/src/connectors/fileScanning/index.ts | 27 +++++++ backend/src/models/File.ts | 21 +++--- backend/src/services/file.ts | 40 ++-------- backend/src/utils/config.ts | 4 + 7 files changed, 138 insertions(+), 46 deletions(-) create mode 100644 backend/src/connectors/fileScanning/Base.ts create mode 100644 backend/src/connectors/fileScanning/clamAv.ts create mode 100644 backend/src/connectors/fileScanning/index.ts diff --git a/backend/config/default.cjs b/backend/config/default.cjs index 4fc184c9b..a95b1ff64 100644 --- a/backend/config/default.cjs +++ b/backend/config/default.cjs @@ -144,7 +144,7 @@ module.exports = { }, avScanning: { - enabled: false, + enabled: true, clamdscan: { host: '127.0.0.1', port: 3310, @@ -205,6 +205,10 @@ module.exports = { audit: { kind: 'silly', }, + + fileScanners: { + kinds: ['clamAV'], + }, }, s3: { diff --git a/backend/src/connectors/fileScanning/Base.ts b/backend/src/connectors/fileScanning/Base.ts new file mode 100644 index 000000000..b6822d2c4 --- /dev/null +++ b/backend/src/connectors/fileScanning/Base.ts @@ -0,0 +1,11 @@ +import { FileInterface, ScanStateKeys } from '../../models/File.js' +export interface FileScanResult { + toolName: string + state: ScanStateKeys + isInfected?: boolean + viruses?: string[] +} + +export abstract class BaseFileScanningConnector { + abstract scan(file: FileInterface) +} diff --git a/backend/src/connectors/fileScanning/clamAv.ts b/backend/src/connectors/fileScanning/clamAv.ts new file mode 100644 index 000000000..c82e888f6 --- /dev/null +++ b/backend/src/connectors/fileScanning/clamAv.ts @@ -0,0 +1,75 @@ +import NodeClam from 'clamscan' +import { Readable } from 'stream' + +import { getObjectStream } from '../../clients/s3.js' +import FileModel, { FileInterfaceDoc, ScanState } from '../../models/File.js' +import log from '../../services/log.js' +import config from '../../utils/config.js' +import { BaseFileScanningConnector, FileScanResult } from './Base.js' + +let av: NodeClam + +export class ClamAvFileScanningConnector extends BaseFileScanningConnector { + constructor() { + super() + } + + async scan(file: FileInterfaceDoc) { + if (!av) { + try { + av = await new NodeClam().init({ clamdscan: config.avScanning.clamdscan }) + } catch (error) { + log.error(error, 'Unable to connect to ClamAV.') + const updatedAvScanArray = [...file.avScan, { toolName: 'Clam AV', state: ScanState.Error }] + await file.update({ $set: { avScan: updatedAvScanArray } }) + } + } + const avStream = av.passthrough() + const s3Stream = (await getObjectStream(file.bucket, file.path)).Body as Readable + s3Stream.pipe(avStream) + log.info({ modelId: file.modelId, fileId: file._id, name: file.name }, 'Scan started.') + const updatedAvScanArray = [...file.avScan, { toolName: 'Clam AV', state: ScanState.InProgress }] + await file.update({ $set: { avScan: updatedAvScanArray } }) + avStream.on('scan-complete', async (result) => { + log.info({ result, modelId: file.modelId, fileId: file._id, name: file.name }, 'Scan complete.') + const newResult: FileScanResult = { + toolName: 'Clam AV', + state: ScanState.Complete, + isInfected: result.isInfected, + viruses: result.viruses, + } + await FileModel.updateOne( + { _id: file._id, 'avScan.toolName': 'Clam AV' }, + { + $set: { 'avScan.$': newResult }, + }, + ) + }) + avStream.on('error', async (error) => { + log.error({ error, modelId: file.modelId, fileId: file._id, name: file.name }, 'Scan errored.') + const newResult: FileScanResult = { + toolName: 'Clam AV', + state: ScanState.Error, + } + await FileModel.updateOne( + { _id: file._id, 'avScan.toolName': 'Clam AV' }, + { + $set: { 'avScan.$': newResult }, + }, + ) + }) + avStream.on('timeout', async (error) => { + log.error({ error, modelId: file.modelId, fileId: file._id, name: file.name }, 'Scan timed out.') + const newResult: FileScanResult = { + toolName: 'Clam AV', + state: ScanState.Error, + } + await FileModel.updateOne( + { _id: file._id, 'avScan.toolName': 'Clam AV' }, + { + $set: { 'avScan.$': newResult }, + }, + ) + }) + } +} diff --git a/backend/src/connectors/fileScanning/index.ts b/backend/src/connectors/fileScanning/index.ts new file mode 100644 index 000000000..da8bf8aee --- /dev/null +++ b/backend/src/connectors/fileScanning/index.ts @@ -0,0 +1,27 @@ +import config from '../../utils/config.js' +import { ConfigurationError } from '../../utils/error.js' +import { BaseFileScanningConnector } from './Base.js' +import { ClamAvFileScanningConnector } from './clamAv.js' + +export const FileScanKind = { + ClamAv: 'clamAV', +} as const +export type FileScanKindKeys = (typeof FileScanKind)[keyof typeof FileScanKind] + +const fileScanConnectors: BaseFileScanningConnector[] = [] +export function getFileScanningConnectors(_cache = true) { + config.connectors.fileScanners.kinds.forEach((fileScanner) => { + switch (fileScanner) { + case FileScanKind.ClamAv: + fileScanConnectors.push(new ClamAvFileScanningConnector()) + break + default: + throw ConfigurationError(`'${fileScanner}' is not a valid file scanning kind.`, { + validKinds: Object.values(FileScanKind), + }) + } + }) + return fileScanConnectors +} + +export default getFileScanningConnectors() diff --git a/backend/src/models/File.ts b/backend/src/models/File.ts index 5100accc9..b4c9dc276 100644 --- a/backend/src/models/File.ts +++ b/backend/src/models/File.ts @@ -1,6 +1,8 @@ import { Document, model, ObjectId, Schema } from 'mongoose' import MongooseDelete from 'mongoose-delete' +import { FileScanResult } from '../connectors/fileScanning/Base.js' + // This interface stores information about the properties on the base object. // It should be used for plain object representations, e.g. for sending to the // client. @@ -17,11 +19,7 @@ export interface FileInterface { complete: boolean - avScan: { - state: ScanStateKeys - isInfected?: boolean - viruses?: Array - } + avScan: Array createdAt: Date updatedAt: Date @@ -51,11 +49,14 @@ const FileSchema = new Schema( bucket: { type: String, required: true }, path: { type: String, required: true }, - avScan: { - state: { type: String, enum: Object.values(ScanState), default: 'notScanned' }, - isInfected: { type: Boolean }, - viruses: [{ type: String }], - }, + avScan: [ + { + toolName: { type: String }, + state: { type: String, enum: Object.values(ScanState) }, + isInfected: { type: Boolean }, + viruses: [{ type: String }], + }, + ], complete: { type: Boolean, default: false }, }, diff --git a/backend/src/services/file.ts b/backend/src/services/file.ts index 9deb287db..6178152f0 100644 --- a/backend/src/services/file.ts +++ b/backend/src/services/file.ts @@ -1,20 +1,17 @@ -import NodeClam from 'clamscan' import { Readable } from 'stream' import { getObjectStream, putObjectStream } from '../clients/s3.js' import { FileAction } from '../connectors/authorisation/actions.js' import authorisation from '../connectors/authorisation/index.js' -import FileModel, { ScanState } from '../models/File.js' +import { getFileScanningConnectors } from '../connectors/fileScanning/index.js' +import FileModel from '../models/File.js' import { UserInterface } from '../models/User.js' import config from '../utils/config.js' import { BadReq, Forbidden, NotFound } from '../utils/error.js' import { longId } from '../utils/id.js' -import log from './log.js' import { getModelById } from './model.js' import { removeFileFromReleases } from './release.js' -let av: NodeClam - export async function uploadFile(user: UserInterface, modelId: string, name: string, mime: string, stream: Readable) { const model = await getModelById(user, modelId) if (model.settings.mirror.sourceModelId) { @@ -39,36 +36,9 @@ export async function uploadFile(user: UserInterface, modelId: string, name: str await file.save() if (config.avScanning.enabled) { - if (!av) { - try { - av = await new NodeClam().init({ clamdscan: config.avScanning.clamdscan }) - } catch (error) { - log.error(error, 'Unable to connect to ClamAV.') - return file - } - } - const avStream = av.passthrough() - const s3Stream = (await getObjectStream(file.bucket, file.path)).Body as Readable - s3Stream.pipe(avStream) - log.info({ modelId, fileId: file._id, name }, 'Scan started.') - - avStream.on('scan-complete', async (result) => { - log.info({ result, modelId, fileId: file._id, name }, 'Scan complete.') - await file.update({ - avScan: { state: ScanState.Complete, isInfected: result.isInfected, viruses: result.viruses }, - }) - }) - avStream.on('error', async (error) => { - log.error({ error, modelId, fileId: file._id, name }, 'Scan errored.') - await file.update({ - avScan: { state: ScanState.Error }, - }) - }) - avStream.on('timeout', async (error) => { - log.error({ error, modelId, fileId: file._id, name }, 'Scan timed out.') - await file.update({ - avScan: { state: ScanState.Error }, - }) + const fileScanConnectors = getFileScanningConnectors() + fileScanConnectors.forEach((fileScanner) => { + fileScanner.scan(file) }) } diff --git a/backend/src/utils/config.ts b/backend/src/utils/config.ts index a0143dff5..d8e4169bb 100644 --- a/backend/src/utils/config.ts +++ b/backend/src/utils/config.ts @@ -36,6 +36,10 @@ export interface Config { audit: { kind: AuditKindKeys } + + fileScanners: { + kinds: string[] + } } smtp: { From 0199b64bb3e2b85e6934aefacbfa7e21fb6bbbe4 Mon Sep 17 00:00:00 2001 From: araddcc002 Date: Wed, 18 Sep 2024 12:39:24 +0000 Subject: [PATCH 02/32] wip for config changes for filescanning --- backend/src/utils/__mocks__/config.ts | 3 +++ backend/src/utils/config.ts | 3 ++- .../helm/bailo/templates/bailo/bailo.configmap.yaml | 4 ++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/backend/src/utils/__mocks__/config.ts b/backend/src/utils/__mocks__/config.ts index c26361d60..673b09f9b 100644 --- a/backend/src/utils/__mocks__/config.ts +++ b/backend/src/utils/__mocks__/config.ts @@ -20,6 +20,9 @@ const config = { authorisation: { kind: 'basic', }, + fileScanners: { + kinds: ['clamAv'], + }, }, log: { level: 'debug', diff --git a/backend/src/utils/config.ts b/backend/src/utils/config.ts index d8e4169bb..cb4fa302f 100644 --- a/backend/src/utils/config.ts +++ b/backend/src/utils/config.ts @@ -5,6 +5,7 @@ import grant from 'grant' import { AuditKindKeys } from '../connectors/audit/index.js' import { AuthenticationKindKeys } from '../connectors/authentication/index.js' import { AuthorisationKindKeys } from '../connectors/authorisation/index.js' +import { FileScanKindKeys } from '../connectors/fileScanning/index.js' import { DefaultSchema } from '../services/schema.js' import { deepFreeze } from './object.js' @@ -38,7 +39,7 @@ export interface Config { } fileScanners: { - kinds: string[] + kinds: FileScanKindKeys[] } } diff --git a/infrastructure/helm/bailo/templates/bailo/bailo.configmap.yaml b/infrastructure/helm/bailo/templates/bailo/bailo.configmap.yaml index 883e1fa34..622eae5f0 100644 --- a/infrastructure/helm/bailo/templates/bailo/bailo.configmap.yaml +++ b/infrastructure/helm/bailo/templates/bailo/bailo.configmap.yaml @@ -189,6 +189,10 @@ data: audit: { kind: '{{ .Values.connectors.audit.kind }}', }, + + fileScanners: { + kinds: '{{ .Values.connectors.fileScanners.kinds }}', + }, }, log: { From cde5410266aa57f9b3bc632286be8fdac6f9994a Mon Sep 17 00:00:00 2001 From: araddcc002 Date: Fri, 20 Sep 2024 10:45:55 +0000 Subject: [PATCH 03/32] updated unit tests --- backend/src/services/mirroredModel.ts | 11 +++++++-- backend/src/utils/__mocks__/config.ts | 2 +- .../services/__snapshots__/file.spec.ts.snap | 17 +------------- backend/test/services/file.spec.ts | 23 +++++++++++++++++++ backend/test/services/mirroredModel.spec.ts | 14 +++++------ 5 files changed, 41 insertions(+), 26 deletions(-) diff --git a/backend/src/services/mirroredModel.ts b/backend/src/services/mirroredModel.ts index a2078f5ad..0e0b52f4d 100644 --- a/backend/src/services/mirroredModel.ts +++ b/backend/src/services/mirroredModel.ts @@ -351,9 +351,16 @@ async function checkReleaseFiles(user: UserInterface, modelId: string, semvers: for (const file of files) { if (!file.avScan) { scanErrors.missingScan.push({ name: file.name, id: file.id }) - } else if (file.avScan.state !== ScanState.Complete) { + } else if ( + file.avScan.some( + (scanResult) => + scanResult.state === ScanState.NotScanned || + scanResult.state === ScanState.Error || + scanResult.state === ScanState.InProgress, + ) + ) { scanErrors.incompleteScan.push({ name: file.name, id: file.id }) - } else if (file.avScan.isInfected) { + } else if (file.avScan.some((scanResult) => scanResult.isInfected)) { scanErrors.failedScan.push({ name: file.name, id: file.id }) } } diff --git a/backend/src/utils/__mocks__/config.ts b/backend/src/utils/__mocks__/config.ts index 673b09f9b..62ca6296b 100644 --- a/backend/src/utils/__mocks__/config.ts +++ b/backend/src/utils/__mocks__/config.ts @@ -21,7 +21,7 @@ const config = { kind: 'basic', }, fileScanners: { - kinds: ['clamAv'], + kinds: ['clamAV'], }, }, log: { diff --git a/backend/test/services/__snapshots__/file.spec.ts.snap b/backend/test/services/__snapshots__/file.spec.ts.snap index a32615be2..21fc76217 100644 --- a/backend/test/services/__snapshots__/file.spec.ts.snap +++ b/backend/test/services/__snapshots__/file.spec.ts.snap @@ -61,19 +61,4 @@ exports[`services > file > removeFile > success 1`] = ` } `; -exports[`services > file > uploadFile > virus scan initialised 1`] = ` -[ - [ - "scan-complete", - [Function], - ], - [ - "error", - [Function], - ], - [ - "timeout", - [Function], - ], -] -`; +exports[`services > file > uploadFile > virus scan initialised 1`] = `[]`; diff --git a/backend/test/services/file.spec.ts b/backend/test/services/file.spec.ts index 4ff8f02e7..4b71ddd36 100644 --- a/backend/test/services/file.spec.ts +++ b/backend/test/services/file.spec.ts @@ -3,6 +3,7 @@ import { describe, expect, test, vi } from 'vitest' import { FileAction } from '../../src/connectors/authorisation/actions.js' import authorisation from '../../src/connectors/authorisation/index.js' +import { ClamAvFileScanningConnector } from '../../src/connectors/fileScanning/clamAv.js' import { UserInterface } from '../../src/models/User.js' import { downloadFile, @@ -14,6 +15,7 @@ import { } from '../../src/services/file.js' vi.mock('../../src/connectors/authorisation/index.js') +vi.mock('../../src/connectors/fileScanning/index.js') const logMock = vi.hoisted(() => ({ info: vi.fn(), @@ -35,6 +37,11 @@ const configMock = vi.hoisted( registry: 'registry', }, }, + connectors: { + fileScanners: { + kinds: ['clamAV'], + }, + }, }) as any, ) vi.mock('../../src/utils/config.js', () => ({ @@ -42,12 +49,22 @@ vi.mock('../../src/utils/config.js', () => ({ default: configMock, })) +const fileScanningConnector = vi.hoisted(() => ({ + getFileScanningConnectors: vi.fn(() => [new ClamAvFileScanningConnector()]), +})) +vi.mock('../../src/connectors/fileScanning/index.js', () => fileScanningConnector) + const s3Mocks = vi.hoisted(() => ({ putObjectStream: vi.fn(() => ({ fileSize: 100 })), getObjectStream: vi.fn(() => ({ Body: { pipe: vi.fn() } })), })) vi.mock('../../src/clients/s3.js', () => s3Mocks) +const clamAvScan = vi.hoisted(() => ({ + scan: vi.fn(() => {}), +})) +vi.mock('../../src/connectors/fileScanning/clamAv.js') + const modelMocks = vi.hoisted(() => ({ getModelById: vi.fn(() => ({ settings: { mirror: { sourceModelId: '' } } })), })) @@ -101,6 +118,12 @@ describe('services > file', () => { test('uploadFile > virus scan initialised', async () => { vi.spyOn(configMock, 'avScanning', 'get').mockReturnValue({ enabled: true, clamdscan: 'test' }) + vi.spyOn(configMock, 'connectors', 'get').mockReturnValue({ + fileScanners: { + kinds: ['clamAV'], + }, + }) + vi.mocked(clamAvScan.scan).mockImplementation(() => {}) const user = { dn: 'testUser' } as UserInterface const modelId = 'testModelId' const name = 'testFile' diff --git a/backend/test/services/mirroredModel.spec.ts b/backend/test/services/mirroredModel.spec.ts index 9cb33a420..41ea90189 100644 --- a/backend/test/services/mirroredModel.spec.ts +++ b/backend/test/services/mirroredModel.spec.ts @@ -80,7 +80,7 @@ const releaseMocks = vi.hoisted(() => ({ vi.mock('../../src/services/release.js', () => releaseMocks) const fileMocks = vi.hoisted(() => ({ - getFilesByIds: vi.fn(() => [{ _id: '123', avScan: { state: 'complete', isInfected: false }, toJSON: vi.fn() }]), + getFilesByIds: vi.fn(() => [{ _id: '123', avScan: [{ state: 'complete', isInfected: false }], toJSON: vi.fn() }]), getTotalFileSize: vi.fn(() => 42), downloadFile: vi.fn(() => ({ Body: 'test' })), })) @@ -218,8 +218,8 @@ describe('services > mirroredModel', () => { test('exportModel > export contains infected file', async () => { fileMocks.getFilesByIds.mockReturnValueOnce([ - { _id: '123', avScan: { state: 'complete', isInfected: true }, toJSON: vi.fn() }, - { _id: '321', avScan: { state: 'complete', isInfected: false }, toJSON: vi.fn() }, + { _id: '123', avScan: [{ state: 'complete', isInfected: true }], toJSON: vi.fn() }, + { _id: '321', avScan: [{ state: 'complete', isInfected: false }], toJSON: vi.fn() }, ]) const response = exportModel({} as UserInterface, 'modelId', true, ['1.2.3']) await expect(response).rejects.toThrowError('The releases contain file(s) that do not have a clean AV scan.') @@ -228,8 +228,8 @@ describe('services > mirroredModel', () => { test('exportModel > export contains incomplete file scan', async () => { fileMocks.getFilesByIds.mockReturnValueOnce([ - { _id: '123', avScan: { state: 'inProgress' } as any, toJSON: vi.fn() }, - { _id: '321', avScan: { state: 'complete', isInfected: false }, toJSON: vi.fn() }, + { _id: '123', avScan: [{ state: 'inProgress' }] as any, toJSON: vi.fn() }, + { _id: '321', avScan: [{ state: 'complete', isInfected: false }], toJSON: vi.fn() }, ]) const response = exportModel({} as UserInterface, 'modelId', true, ['1.2.3']) await expect(response).rejects.toThrowError('The releases contain file(s) that do not have a clean AV scan.') @@ -239,8 +239,8 @@ describe('services > mirroredModel', () => { test('exportModel > export missing file scan', async () => { fileMocks.getFilesByIds.mockReturnValueOnce([ { _id: '123', toJSON: vi.fn() } as any, - { _id: '321', avScan: { state: 'complete', isInfected: false }, toJSON: vi.fn() }, - { _id: '321', avScan: { state: 'complete', isInfected: false }, toJSON: vi.fn() }, + { _id: '321', avScan: [{ state: 'complete', isInfected: false }], toJSON: vi.fn() }, + { _id: '321', avScan: [{ state: 'complete', isInfected: false }], toJSON: vi.fn() }, ]) const response = exportModel({} as UserInterface, 'testmod', true, ['1.2.3']) await expect(response).rejects.toThrowError('The releases contain file(s) that do not have a clean AV scan.') From 93560e16ee2376e469112162a94aa2ca3def0839 Mon Sep 17 00:00:00 2001 From: araddcc002 Date: Wed, 25 Sep 2024 12:26:54 +0000 Subject: [PATCH 04/32] updated av scanning config --- .../helm/bailo/templates/bailo/bailo.configmap.yaml | 4 ++++ infrastructure/helm/bailo/values.yaml | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/infrastructure/helm/bailo/templates/bailo/bailo.configmap.yaml b/infrastructure/helm/bailo/templates/bailo/bailo.configmap.yaml index 622eae5f0..2d8e0ba9a 100644 --- a/infrastructure/helm/bailo/templates/bailo/bailo.configmap.yaml +++ b/infrastructure/helm/bailo/templates/bailo/bailo.configmap.yaml @@ -175,6 +175,10 @@ data: enabled: {{ .Values.modelMirror.enabled }}, disclaimer: '{{ .Values.modelMirror.disclaimer }}' } + + avScanning: { + enabled: {{ .Values.config.ui.avScanning.enabled }}, + } }, connectors: { diff --git a/infrastructure/helm/bailo/values.yaml b/infrastructure/helm/bailo/values.yaml index c4c935757..523300ce2 100644 --- a/infrastructure/helm/bailo/values.yaml +++ b/infrastructure/helm/bailo/values.yaml @@ -230,6 +230,9 @@ config: text: '' startTimestamp: '' + avScanning: + enabled: false + smtp: #host: 'mail' #service name port: 1025 @@ -258,6 +261,9 @@ connectors: audit: kind: 'silly' + fileScanning: + kinds: ['clamAV'] + instrumentation: enabled: false debug: false From 801b5a7f00cf018b3697f07d36bdf2e669144e18 Mon Sep 17 00:00:00 2001 From: araddcc002 Date: Wed, 25 Sep 2024 12:35:24 +0000 Subject: [PATCH 05/32] updated av scanning config --- infrastructure/helm/bailo/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/helm/bailo/values.yaml b/infrastructure/helm/bailo/values.yaml index 523300ce2..91786fc38 100644 --- a/infrastructure/helm/bailo/values.yaml +++ b/infrastructure/helm/bailo/values.yaml @@ -261,7 +261,7 @@ connectors: audit: kind: 'silly' - fileScanning: + fileScanners: kinds: ['clamAV'] instrumentation: From a00b6c7267a324bc1735386d8ef2cd579d2ad9e6 Mon Sep 17 00:00:00 2001 From: araddcc002 Date: Fri, 27 Sep 2024 07:43:17 +0000 Subject: [PATCH 06/32] updated config to disable file scanning by default --- backend/config/default.cjs | 2 +- backend/config/docker_compose.cjs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/config/default.cjs b/backend/config/default.cjs index a2db78e10..9dec4e023 100644 --- a/backend/config/default.cjs +++ b/backend/config/default.cjs @@ -192,7 +192,7 @@ module.exports = { }, avScanning: { - enabled: true, + enabled: false, }, }, diff --git a/backend/config/docker_compose.cjs b/backend/config/docker_compose.cjs index 3b89c9181..6edd987bf 100644 --- a/backend/config/docker_compose.cjs +++ b/backend/config/docker_compose.cjs @@ -69,7 +69,7 @@ module.exports = { ui: { avScanning: { - enabled: true, + enabled: false, }, }, } From 7244593f78938253d6564aa65c9d223d1daee593 Mon Sep 17 00:00:00 2001 From: araddcc002 Date: Fri, 27 Sep 2024 10:23:03 +0000 Subject: [PATCH 07/32] added check for required config --- backend/config/docker_compose.cjs | 2 +- backend/src/connectors/fileScanning/clamAv.ts | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/config/docker_compose.cjs b/backend/config/docker_compose.cjs index 6edd987bf..3b89c9181 100644 --- a/backend/config/docker_compose.cjs +++ b/backend/config/docker_compose.cjs @@ -69,7 +69,7 @@ module.exports = { ui: { avScanning: { - enabled: false, + enabled: true, }, }, } diff --git a/backend/src/connectors/fileScanning/clamAv.ts b/backend/src/connectors/fileScanning/clamAv.ts index c82e888f6..cee63a43f 100644 --- a/backend/src/connectors/fileScanning/clamAv.ts +++ b/backend/src/connectors/fileScanning/clamAv.ts @@ -11,6 +11,10 @@ let av: NodeClam export class ClamAvFileScanningConnector extends BaseFileScanningConnector { constructor() { + if (!config.avScanning.clamdscan) { + log.error('Unable to fetch Clam AV details from config') + return + } super() } From 82a492ec873436f1e0aed252e55c6e725c5ba80c Mon Sep 17 00:00:00 2001 From: araddcc002 Date: Fri, 27 Sep 2024 11:43:40 +0000 Subject: [PATCH 08/32] changes to file scanning connector --- backend/src/connectors/fileScanning/Base.ts | 1 + backend/src/connectors/fileScanning/clamAv.ts | 11 ++++------- backend/src/connectors/fileScanning/index.ts | 6 +++++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/backend/src/connectors/fileScanning/Base.ts b/backend/src/connectors/fileScanning/Base.ts index b6822d2c4..e0dabe245 100644 --- a/backend/src/connectors/fileScanning/Base.ts +++ b/backend/src/connectors/fileScanning/Base.ts @@ -7,5 +7,6 @@ export interface FileScanResult { } export abstract class BaseFileScanningConnector { + abstract init() abstract scan(file: FileInterface) } diff --git a/backend/src/connectors/fileScanning/clamAv.ts b/backend/src/connectors/fileScanning/clamAv.ts index cee63a43f..c7cde64fa 100644 --- a/backend/src/connectors/fileScanning/clamAv.ts +++ b/backend/src/connectors/fileScanning/clamAv.ts @@ -11,23 +11,20 @@ let av: NodeClam export class ClamAvFileScanningConnector extends BaseFileScanningConnector { constructor() { - if (!config.avScanning.clamdscan) { - log.error('Unable to fetch Clam AV details from config') - return - } super() } - async scan(file: FileInterfaceDoc) { + async init() { if (!av) { try { av = await new NodeClam().init({ clamdscan: config.avScanning.clamdscan }) } catch (error) { log.error(error, 'Unable to connect to ClamAV.') - const updatedAvScanArray = [...file.avScan, { toolName: 'Clam AV', state: ScanState.Error }] - await file.update({ $set: { avScan: updatedAvScanArray } }) } } + } + + async scan(file: FileInterfaceDoc) { const avStream = av.passthrough() const s3Stream = (await getObjectStream(file.bucket, file.path)).Body as Readable s3Stream.pipe(avStream) diff --git a/backend/src/connectors/fileScanning/index.ts b/backend/src/connectors/fileScanning/index.ts index da8bf8aee..b3c51d25a 100644 --- a/backend/src/connectors/fileScanning/index.ts +++ b/backend/src/connectors/fileScanning/index.ts @@ -8,12 +8,16 @@ export const FileScanKind = { } as const export type FileScanKindKeys = (typeof FileScanKind)[keyof typeof FileScanKind] +let fileScanningConnector: undefined | BaseFileScanningConnector = undefined + const fileScanConnectors: BaseFileScanningConnector[] = [] export function getFileScanningConnectors(_cache = true) { config.connectors.fileScanners.kinds.forEach((fileScanner) => { switch (fileScanner) { case FileScanKind.ClamAv: - fileScanConnectors.push(new ClamAvFileScanningConnector()) + fileScanningConnector = new ClamAvFileScanningConnector() + fileScanConnectors.push(fileScanningConnector) + fileScanningConnector.init() break default: throw ConfigurationError(`'${fileScanner}' is not a valid file scanning kind.`, { From 48bc81c1388c5973a3effde400575574186b2c2d Mon Sep 17 00:00:00 2001 From: araddcc002 Date: Fri, 27 Sep 2024 12:07:15 +0000 Subject: [PATCH 09/32] added error checking for clamav config --- backend/src/connectors/fileScanning/clamAv.ts | 7 +++++-- backend/src/connectors/fileScanning/index.ts | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/backend/src/connectors/fileScanning/clamAv.ts b/backend/src/connectors/fileScanning/clamAv.ts index c7cde64fa..5f2fd535f 100644 --- a/backend/src/connectors/fileScanning/clamAv.ts +++ b/backend/src/connectors/fileScanning/clamAv.ts @@ -5,6 +5,7 @@ import { getObjectStream } from '../../clients/s3.js' import FileModel, { FileInterfaceDoc, ScanState } from '../../models/File.js' import log from '../../services/log.js' import config from '../../utils/config.js' +import { ConfigurationError } from '../../utils/error.js' import { BaseFileScanningConnector, FileScanResult } from './Base.js' let av: NodeClam @@ -15,11 +16,13 @@ export class ClamAvFileScanningConnector extends BaseFileScanningConnector { } async init() { - if (!av) { + if (av) { try { av = await new NodeClam().init({ clamdscan: config.avScanning.clamdscan }) } catch (error) { - log.error(error, 'Unable to connect to ClamAV.') + throw ConfigurationError('Cannot retrieve Clam AV config', { + clamAvConfig: config.avScanning, + }) } } } diff --git a/backend/src/connectors/fileScanning/index.ts b/backend/src/connectors/fileScanning/index.ts index b3c51d25a..3dc7b38ca 100644 --- a/backend/src/connectors/fileScanning/index.ts +++ b/backend/src/connectors/fileScanning/index.ts @@ -12,12 +12,12 @@ let fileScanningConnector: undefined | BaseFileScanningConnector = undefined const fileScanConnectors: BaseFileScanningConnector[] = [] export function getFileScanningConnectors(_cache = true) { - config.connectors.fileScanners.kinds.forEach((fileScanner) => { + config.connectors.fileScanners.kinds.forEach(async (fileScanner) => { switch (fileScanner) { case FileScanKind.ClamAv: fileScanningConnector = new ClamAvFileScanningConnector() + await fileScanningConnector.init() fileScanConnectors.push(fileScanningConnector) - fileScanningConnector.init() break default: throw ConfigurationError(`'${fileScanner}' is not a valid file scanning kind.`, { From dbe8651c4d7b7c34b8c675103ba97fd0084622ac Mon Sep 17 00:00:00 2001 From: araddcc002 Date: Fri, 27 Sep 2024 12:15:03 +0000 Subject: [PATCH 10/32] updated docker compose prod file --- docker-compose-prod.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker-compose-prod.yml b/docker-compose-prod.yml index 505f4d363..5aa68c551 100644 --- a/docker-compose-prod.yml +++ b/docker-compose-prod.yml @@ -25,6 +25,9 @@ services: volumes: - minioVolume:/bitnami/minio + clamd: + image: clamav/clamav:1.3 + mailcrab: image: marlonb/mailcrab:v1.2.0 ports: From 57d1fb05af8c75e2ebdaaafd3330eea3eda1fe60 Mon Sep 17 00:00:00 2001 From: araddcc002 Date: Fri, 27 Sep 2024 12:27:11 +0000 Subject: [PATCH 11/32] added awaits for getfilescanconnectors call --- backend/src/connectors/fileScanning/index.ts | 2 +- backend/src/services/file.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/connectors/fileScanning/index.ts b/backend/src/connectors/fileScanning/index.ts index 3dc7b38ca..57e69f516 100644 --- a/backend/src/connectors/fileScanning/index.ts +++ b/backend/src/connectors/fileScanning/index.ts @@ -11,7 +11,7 @@ export type FileScanKindKeys = (typeof FileScanKind)[keyof typeof FileScanKind] let fileScanningConnector: undefined | BaseFileScanningConnector = undefined const fileScanConnectors: BaseFileScanningConnector[] = [] -export function getFileScanningConnectors(_cache = true) { +export async function getFileScanningConnectors(_cache = true) { config.connectors.fileScanners.kinds.forEach(async (fileScanner) => { switch (fileScanner) { case FileScanKind.ClamAv: diff --git a/backend/src/services/file.ts b/backend/src/services/file.ts index 50f23eea7..8343406c1 100644 --- a/backend/src/services/file.ts +++ b/backend/src/services/file.ts @@ -36,7 +36,7 @@ export async function uploadFile(user: UserInterface, modelId: string, name: str await file.save() if (config.ui.avScanning.enabled) { - const fileScanConnectors = getFileScanningConnectors() + const fileScanConnectors = await getFileScanningConnectors() fileScanConnectors.forEach((fileScanner) => { fileScanner.scan(file) }) From 42fc570bd06dc87ef4f617d404edce52f55e69ab Mon Sep 17 00:00:00 2001 From: araddcc002 Date: Fri, 27 Sep 2024 12:30:04 +0000 Subject: [PATCH 12/32] added awaits for getfilescanconnectors call --- backend/src/connectors/fileScanning/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/connectors/fileScanning/index.ts b/backend/src/connectors/fileScanning/index.ts index 57e69f516..8c0bc1536 100644 --- a/backend/src/connectors/fileScanning/index.ts +++ b/backend/src/connectors/fileScanning/index.ts @@ -16,7 +16,6 @@ export async function getFileScanningConnectors(_cache = true) { switch (fileScanner) { case FileScanKind.ClamAv: fileScanningConnector = new ClamAvFileScanningConnector() - await fileScanningConnector.init() fileScanConnectors.push(fileScanningConnector) break default: @@ -24,6 +23,7 @@ export async function getFileScanningConnectors(_cache = true) { validKinds: Object.values(FileScanKind), }) } + await fileScanningConnector.init() }) return fileScanConnectors } From 607bd6bcef178eec0959014f401845565849891a Mon Sep 17 00:00:00 2001 From: araddcc002 Date: Fri, 27 Sep 2024 12:53:13 +0000 Subject: [PATCH 13/32] reverted logic for checking if av is initialised --- backend/src/connectors/fileScanning/clamAv.ts | 2 +- backend/src/connectors/fileScanning/index.ts | 10 +++------- backend/src/services/file.ts | 7 ++++--- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/backend/src/connectors/fileScanning/clamAv.ts b/backend/src/connectors/fileScanning/clamAv.ts index 5f2fd535f..2a95c17df 100644 --- a/backend/src/connectors/fileScanning/clamAv.ts +++ b/backend/src/connectors/fileScanning/clamAv.ts @@ -16,7 +16,7 @@ export class ClamAvFileScanningConnector extends BaseFileScanningConnector { } async init() { - if (av) { + if (!av) { try { av = await new NodeClam().init({ clamdscan: config.avScanning.clamdscan }) } catch (error) { diff --git a/backend/src/connectors/fileScanning/index.ts b/backend/src/connectors/fileScanning/index.ts index 8c0bc1536..da8bf8aee 100644 --- a/backend/src/connectors/fileScanning/index.ts +++ b/backend/src/connectors/fileScanning/index.ts @@ -8,22 +8,18 @@ export const FileScanKind = { } as const export type FileScanKindKeys = (typeof FileScanKind)[keyof typeof FileScanKind] -let fileScanningConnector: undefined | BaseFileScanningConnector = undefined - const fileScanConnectors: BaseFileScanningConnector[] = [] -export async function getFileScanningConnectors(_cache = true) { - config.connectors.fileScanners.kinds.forEach(async (fileScanner) => { +export function getFileScanningConnectors(_cache = true) { + config.connectors.fileScanners.kinds.forEach((fileScanner) => { switch (fileScanner) { case FileScanKind.ClamAv: - fileScanningConnector = new ClamAvFileScanningConnector() - fileScanConnectors.push(fileScanningConnector) + fileScanConnectors.push(new ClamAvFileScanningConnector()) break default: throw ConfigurationError(`'${fileScanner}' is not a valid file scanning kind.`, { validKinds: Object.values(FileScanKind), }) } - await fileScanningConnector.init() }) return fileScanConnectors } diff --git a/backend/src/services/file.ts b/backend/src/services/file.ts index 8343406c1..b1c31b10a 100644 --- a/backend/src/services/file.ts +++ b/backend/src/services/file.ts @@ -36,9 +36,10 @@ export async function uploadFile(user: UserInterface, modelId: string, name: str await file.save() if (config.ui.avScanning.enabled) { - const fileScanConnectors = await getFileScanningConnectors() - fileScanConnectors.forEach((fileScanner) => { - fileScanner.scan(file) + const fileScanConnectors = getFileScanningConnectors() + fileScanConnectors.forEach(async (fileScanner) => { + await fileScanner.init() + await fileScanner.scan(file) }) } From 5c5bcde4186e36c0ec3c2882700df1b1acdcab2b Mon Sep 17 00:00:00 2001 From: araddcc002 Date: Fri, 27 Sep 2024 13:33:38 +0000 Subject: [PATCH 14/32] added the file av results popover for successful av scans --- .../src/entry/model/releases/FileDownload.tsx | 96 +++++++++---------- 1 file changed, 45 insertions(+), 51 deletions(-) diff --git a/frontend/src/entry/model/releases/FileDownload.tsx b/frontend/src/entry/model/releases/FileDownload.tsx index 94a462edd..db959dec1 100644 --- a/frontend/src/entry/model/releases/FileDownload.tsx +++ b/frontend/src/entry/model/releases/FileDownload.tsx @@ -28,67 +28,61 @@ export default function FileDownload({ modelId, file }: FileDownloadProps) { ) { return } - const allTestsPassed = file.avScan.every((scan) => !scan.isInfected) const threatsFound = file.avScan.reduce((acc, scan) => { return scan.viruses ? scan.viruses.length + acc : acc }, 0) if (file.avScan.some((scan) => scan.state !== ScanState.Complete)) { return } - if (threatsFound) { - return ( - <> - } - size='small' - onClick={(e) => setAnchorEl(e.currentTarget)} - label={`Virus scan failed: ${plural(threatsFound, 'threat')} found`} - /> - setAnchorEl(null)} - anchorOrigin={{ - vertical: 'bottom', - horizontal: 'center', - }} - transformOrigin={{ - vertical: 'top', - horizontal: 'center', - }} - > - }> - {file.avScan.map((scanResult) => ( - - {scanResult.isInfected ? ( - <> - - - - {scanResult.toolName} found the following threats: - - -
    {scanResult.viruses && scanResult.viruses.map((virus) =>
  • {virus}
    • )}
- - ) : ( + return ( + <> + : } + size='small' + onClick={(e) => setAnchorEl(e.currentTarget)} + label={threatsFound ? `Virus scan failed: ${plural(threatsFound, 'threat')} found` : 'Virus scan passed'} + /> + setAnchorEl(null)} + anchorOrigin={{ + vertical: 'bottom', + horizontal: 'center', + }} + transformOrigin={{ + vertical: 'top', + horizontal: 'center', + }} + > + }> + {file.avScan.map((scanResult) => ( + + {scanResult.isInfected ? ( + <> - + - {scanResult.toolName} did not find any threats + {scanResult.toolName} found the following threats: - )} - - ))} - - - - ) - } - if (allTestsPassed) { - return } size='small' label={'Virus scan passed'} /> - } +
    {scanResult.viruses && scanResult.viruses.map((virus) =>
  • {virus}
    • )}
+ + ) : ( + + + + {scanResult.toolName} did not find any threats + + + )} + + ))} + + + + ) }, [anchorEl, file, open]) if (isUiConfigError) { From 358d87e43a352fc12a53765f6f467cd012a3296b Mon Sep 17 00:00:00 2001 From: araddcc002 Date: Tue, 1 Oct 2024 11:35:16 +0000 Subject: [PATCH 15/32] fixed an issue that was causing only one file scan to store results --- backend/src/connectors/fileScanning/clamAv.ts | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/backend/src/connectors/fileScanning/clamAv.ts b/backend/src/connectors/fileScanning/clamAv.ts index 2a95c17df..502627a36 100644 --- a/backend/src/connectors/fileScanning/clamAv.ts +++ b/backend/src/connectors/fileScanning/clamAv.ts @@ -9,6 +9,7 @@ import { ConfigurationError } from '../../utils/error.js' import { BaseFileScanningConnector, FileScanResult } from './Base.js' let av: NodeClam +const toolName = 'Clam AV' export class ClamAvFileScanningConnector extends BaseFileScanningConnector { constructor() { @@ -32,18 +33,26 @@ export class ClamAvFileScanningConnector extends BaseFileScanningConnector { const s3Stream = (await getObjectStream(file.bucket, file.path)).Body as Readable s3Stream.pipe(avStream) log.info({ modelId: file.modelId, fileId: file._id, name: file.name }, 'Scan started.') - const updatedAvScanArray = [...file.avScan, { toolName: 'Clam AV', state: ScanState.InProgress }] - await file.update({ $set: { avScan: updatedAvScanArray } }) + if (!file.avScan.find((result) => result.toolName === toolName)) { + const updatedAvScanArray = [...file.avScan, { toolName: toolName, state: ScanState.InProgress }] + file.avScan = updatedAvScanArray + await FileModel.updateOne( + { _id: file._id }, + { + $set: { avScan: updatedAvScanArray }, + }, + ) + } avStream.on('scan-complete', async (result) => { log.info({ result, modelId: file.modelId, fileId: file._id, name: file.name }, 'Scan complete.') const newResult: FileScanResult = { - toolName: 'Clam AV', + toolName: toolName, state: ScanState.Complete, isInfected: result.isInfected, viruses: result.viruses, } await FileModel.updateOne( - { _id: file._id, 'avScan.toolName': 'Clam AV' }, + { _id: file._id, 'avScan.toolName': toolName }, { $set: { 'avScan.$': newResult }, }, @@ -52,11 +61,11 @@ export class ClamAvFileScanningConnector extends BaseFileScanningConnector { avStream.on('error', async (error) => { log.error({ error, modelId: file.modelId, fileId: file._id, name: file.name }, 'Scan errored.') const newResult: FileScanResult = { - toolName: 'Clam AV', + toolName: toolName, state: ScanState.Error, } await FileModel.updateOne( - { _id: file._id, 'avScan.toolName': 'Clam AV' }, + { _id: file._id, 'avScan.toolName': toolName }, { $set: { 'avScan.$': newResult }, }, @@ -65,11 +74,11 @@ export class ClamAvFileScanningConnector extends BaseFileScanningConnector { avStream.on('timeout', async (error) => { log.error({ error, modelId: file.modelId, fileId: file._id, name: file.name }, 'Scan timed out.') const newResult: FileScanResult = { - toolName: 'Clam AV', + toolName: toolName, state: ScanState.Error, } await FileModel.updateOne( - { _id: file._id, 'avScan.toolName': 'Clam AV' }, + { _id: file._id, 'avScan.toolName': toolName }, { $set: { 'avScan.$': newResult }, }, From c577734e41f9452daf416f58a6112768b44ab6c7 Mon Sep 17 00:00:00 2001 From: araddcc002 Date: Wed, 2 Oct 2024 11:01:30 +0000 Subject: [PATCH 16/32] fixed some pr comments --- backend/src/connectors/fileScanning/Base.ts | 1 - backend/src/connectors/fileScanning/clamAv.ts | 23 ++++++++++++------- backend/src/services/file.ts | 1 - 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/backend/src/connectors/fileScanning/Base.ts b/backend/src/connectors/fileScanning/Base.ts index e0dabe245..b6822d2c4 100644 --- a/backend/src/connectors/fileScanning/Base.ts +++ b/backend/src/connectors/fileScanning/Base.ts @@ -7,6 +7,5 @@ export interface FileScanResult { } export abstract class BaseFileScanningConnector { - abstract init() abstract scan(file: FileInterface) } diff --git a/backend/src/connectors/fileScanning/clamAv.ts b/backend/src/connectors/fileScanning/clamAv.ts index 502627a36..439bac86e 100644 --- a/backend/src/connectors/fileScanning/clamAv.ts +++ b/backend/src/connectors/fileScanning/clamAv.ts @@ -17,18 +17,25 @@ export class ClamAvFileScanningConnector extends BaseFileScanningConnector { } async init() { - if (!av) { - try { - av = await new NodeClam().init({ clamdscan: config.avScanning.clamdscan }) - } catch (error) { - throw ConfigurationError('Cannot retrieve Clam AV config', { - clamAvConfig: config.avScanning, - }) - } + try { + av = await new NodeClam().init({ clamdscan: config.avScanning.clamdscan }) + } catch (error) { + throw ConfigurationError('Could not scan file as Clam AV is not running.', { + clamAvConfig: config.avScanning, + }) } } async scan(file: FileInterfaceDoc) { + await this.init() + if (!av) { + throw ConfigurationError( + 'Clam AV does not look like it is running. Chec that it has been correctly initialised by calling the init function.', + { + clamAvConfig: config.avScanning, + }, + ) + } const avStream = av.passthrough() const s3Stream = (await getObjectStream(file.bucket, file.path)).Body as Readable s3Stream.pipe(avStream) diff --git a/backend/src/services/file.ts b/backend/src/services/file.ts index b1c31b10a..2c8fa2553 100644 --- a/backend/src/services/file.ts +++ b/backend/src/services/file.ts @@ -38,7 +38,6 @@ export async function uploadFile(user: UserInterface, modelId: string, name: str if (config.ui.avScanning.enabled) { const fileScanConnectors = getFileScanningConnectors() fileScanConnectors.forEach(async (fileScanner) => { - await fileScanner.init() await fileScanner.scan(file) }) } From 46b6bba65a7ce25600a7bec0fac215e9dbb550f3 Mon Sep 17 00:00:00 2001 From: araddcc002 Date: Wed, 2 Oct 2024 13:14:07 +0000 Subject: [PATCH 17/32] changed structure of av file scan --- backend/src/connectors/fileScanning/clamAv.ts | 57 +++++++------------ backend/src/connectors/fileScanning/index.ts | 11 ++-- backend/src/services/file.ts | 29 ++++++++-- 3 files changed, 52 insertions(+), 45 deletions(-) diff --git a/backend/src/connectors/fileScanning/clamAv.ts b/backend/src/connectors/fileScanning/clamAv.ts index 439bac86e..a8139044f 100644 --- a/backend/src/connectors/fileScanning/clamAv.ts +++ b/backend/src/connectors/fileScanning/clamAv.ts @@ -2,14 +2,15 @@ import NodeClam from 'clamscan' import { Readable } from 'stream' import { getObjectStream } from '../../clients/s3.js' -import FileModel, { FileInterfaceDoc, ScanState } from '../../models/File.js' +import { FileInterfaceDoc, ScanState } from '../../models/File.js' +import { updateFileWithResults } from '../../services/file.js' import log from '../../services/log.js' import config from '../../utils/config.js' import { ConfigurationError } from '../../utils/error.js' -import { BaseFileScanningConnector, FileScanResult } from './Base.js' +import { BaseFileScanningConnector } from './Base.js' let av: NodeClam -const toolName = 'Clam AV' +export const clamAvToolName = 'Clam AV' export class ClamAvFileScanningConnector extends BaseFileScanningConnector { constructor() { @@ -40,55 +41,39 @@ export class ClamAvFileScanningConnector extends BaseFileScanningConnector { const s3Stream = (await getObjectStream(file.bucket, file.path)).Body as Readable s3Stream.pipe(avStream) log.info({ modelId: file.modelId, fileId: file._id, name: file.name }, 'Scan started.') - if (!file.avScan.find((result) => result.toolName === toolName)) { - const updatedAvScanArray = [...file.avScan, { toolName: toolName, state: ScanState.InProgress }] - file.avScan = updatedAvScanArray - await FileModel.updateOne( - { _id: file._id }, - { - $set: { avScan: updatedAvScanArray }, - }, - ) - } avStream.on('scan-complete', async (result) => { log.info({ result, modelId: file.modelId, fileId: file._id, name: file.name }, 'Scan complete.') - const newResult: FileScanResult = { - toolName: toolName, - state: ScanState.Complete, - isInfected: result.isInfected, - viruses: result.viruses, - } - await FileModel.updateOne( - { _id: file._id, 'avScan.toolName': toolName }, + updateFileWithResults( + file, { - $set: { 'avScan.$': newResult }, + toolName: clamAvToolName, + state: ScanState.Complete, + isInfected: result.isInfected, + viruses: result.viruses, }, + clamAvToolName, ) }) avStream.on('error', async (error) => { log.error({ error, modelId: file.modelId, fileId: file._id, name: file.name }, 'Scan errored.') - const newResult: FileScanResult = { - toolName: toolName, - state: ScanState.Error, - } - await FileModel.updateOne( - { _id: file._id, 'avScan.toolName': toolName }, + updateFileWithResults( + file, { - $set: { 'avScan.$': newResult }, + toolName: clamAvToolName, + state: ScanState.Error, }, + clamAvToolName, ) }) avStream.on('timeout', async (error) => { log.error({ error, modelId: file.modelId, fileId: file._id, name: file.name }, 'Scan timed out.') - const newResult: FileScanResult = { - toolName: toolName, - state: ScanState.Error, - } - await FileModel.updateOne( - { _id: file._id, 'avScan.toolName': toolName }, + updateFileWithResults( + file, { - $set: { 'avScan.$': newResult }, + toolName: clamAvToolName, + state: ScanState.Error, }, + clamAvToolName, ) }) } diff --git a/backend/src/connectors/fileScanning/index.ts b/backend/src/connectors/fileScanning/index.ts index da8bf8aee..542d407fd 100644 --- a/backend/src/connectors/fileScanning/index.ts +++ b/backend/src/connectors/fileScanning/index.ts @@ -1,7 +1,9 @@ +import { FileInterface, ScanState } from '../../models/File.js' +import { updateFileWithResults } from '../../services/file.js' import config from '../../utils/config.js' import { ConfigurationError } from '../../utils/error.js' import { BaseFileScanningConnector } from './Base.js' -import { ClamAvFileScanningConnector } from './clamAv.js' +import { ClamAvFileScanningConnector, clamAvToolName } from './clamAv.js' export const FileScanKind = { ClamAv: 'clamAV', @@ -9,11 +11,12 @@ export const FileScanKind = { export type FileScanKindKeys = (typeof FileScanKind)[keyof typeof FileScanKind] const fileScanConnectors: BaseFileScanningConnector[] = [] -export function getFileScanningConnectors(_cache = true) { - config.connectors.fileScanners.kinds.forEach((fileScanner) => { +export function getFileScanningConnectors(file: FileInterface) { + config.connectors.fileScanners.kinds.forEach(async (fileScanner) => { switch (fileScanner) { case FileScanKind.ClamAv: fileScanConnectors.push(new ClamAvFileScanningConnector()) + await updateFileWithResults(file, { toolName: clamAvToolName, state: ScanState.InProgress }, clamAvToolName) break default: throw ConfigurationError(`'${fileScanner}' is not a valid file scanning kind.`, { @@ -23,5 +26,3 @@ export function getFileScanningConnectors(_cache = true) { }) return fileScanConnectors } - -export default getFileScanningConnectors() diff --git a/backend/src/services/file.ts b/backend/src/services/file.ts index 2c8fa2553..dfd9b9b4d 100644 --- a/backend/src/services/file.ts +++ b/backend/src/services/file.ts @@ -3,8 +3,9 @@ import { Readable } from 'stream' import { getObjectStream, putObjectStream } from '../clients/s3.js' import { FileAction } from '../connectors/authorisation/actions.js' import authorisation from '../connectors/authorisation/index.js' +import { FileScanResult } from '../connectors/fileScanning/Base.js' import { getFileScanningConnectors } from '../connectors/fileScanning/index.js' -import FileModel from '../models/File.js' +import FileModel, { FileInterface, ScanState } from '../models/File.js' import { UserInterface } from '../models/User.js' import config from '../utils/config.js' import { BadReq, Forbidden, NotFound } from '../utils/error.js' @@ -36,15 +37,35 @@ export async function uploadFile(user: UserInterface, modelId: string, name: str await file.save() if (config.ui.avScanning.enabled) { - const fileScanConnectors = getFileScanningConnectors() - fileScanConnectors.forEach(async (fileScanner) => { - await fileScanner.scan(file) + const scanners = getFileScanningConnectors(file) + scanners.forEach((scanner) => { + scanner.scan(file) }) } return file } +export async function updateFileWithResults(file: FileInterface, result: FileScanResult, toolName) { + if (!file.avScan.find((result) => result.toolName === toolName)) { + const updatedAvScanArray = [...file.avScan, { toolName: toolName, state: ScanState.InProgress }] + file.avScan = updatedAvScanArray + await FileModel.updateOne( + { _id: file._id }, + { + $set: { avScan: updatedAvScanArray }, + }, + ) + } else { + await FileModel.updateOne( + { _id: file._id, 'avScan.toolName': toolName }, + { + $set: { 'avScan.$': result }, + }, + ) + } +} + export async function downloadFile(user: UserInterface, fileId: string, range?: { start: number; end: number }) { const file = await getFileById(user, fileId) const model = await getModelById(user, file.modelId) From 19abdc47d5bd16d5a0ecf96842b898ecdbb3a245 Mon Sep 17 00:00:00 2001 From: araddcc002 Date: Thu, 10 Oct 2024 16:14:33 +0000 Subject: [PATCH 18/32] various improvements to how we handle the AV connector. Added info end point for checkig what filescanners are active --- backend/src/connectors/fileScanning/Base.ts | 3 +- backend/src/connectors/fileScanning/clamAv.ts | 65 +++++++++---------- backend/src/connectors/fileScanning/index.ts | 12 ++-- .../src/connectors/fileScanning/wrapper.ts | 29 +++++++++ backend/src/routes.ts | 3 + .../v2/filescanning/getFilescanningInfo.ts | 15 +++++ backend/src/services/file.ts | 47 +++++++------- 7 files changed, 109 insertions(+), 65 deletions(-) create mode 100644 backend/src/connectors/fileScanning/wrapper.ts create mode 100644 backend/src/routes/v2/filescanning/getFilescanningInfo.ts diff --git a/backend/src/connectors/fileScanning/Base.ts b/backend/src/connectors/fileScanning/Base.ts index b6822d2c4..2ea4bbd77 100644 --- a/backend/src/connectors/fileScanning/Base.ts +++ b/backend/src/connectors/fileScanning/Base.ts @@ -7,5 +7,6 @@ export interface FileScanResult { } export abstract class BaseFileScanningConnector { - abstract scan(file: FileInterface) + abstract info(): string[] + abstract scan(file: FileInterface): Promise } diff --git a/backend/src/connectors/fileScanning/clamAv.ts b/backend/src/connectors/fileScanning/clamAv.ts index a8139044f..d4504af08 100644 --- a/backend/src/connectors/fileScanning/clamAv.ts +++ b/backend/src/connectors/fileScanning/clamAv.ts @@ -3,11 +3,10 @@ import { Readable } from 'stream' import { getObjectStream } from '../../clients/s3.js' import { FileInterfaceDoc, ScanState } from '../../models/File.js' -import { updateFileWithResults } from '../../services/file.js' import log from '../../services/log.js' import config from '../../utils/config.js' import { ConfigurationError } from '../../utils/error.js' -import { BaseFileScanningConnector } from './Base.js' +import { BaseFileScanningConnector, FileScanResult } from './Base.js' let av: NodeClam export const clamAvToolName = 'Clam AV' @@ -17,6 +16,10 @@ export class ClamAvFileScanningConnector extends BaseFileScanningConnector { super() } + info() { + return [clamAvToolName] + } + async init() { try { av = await new NodeClam().init({ clamdscan: config.avScanning.clamdscan }) @@ -27,7 +30,7 @@ export class ClamAvFileScanningConnector extends BaseFileScanningConnector { } } - async scan(file: FileInterfaceDoc) { + async scan(file: FileInterfaceDoc): Promise { await this.init() if (!av) { throw ConfigurationError( @@ -41,40 +44,32 @@ export class ClamAvFileScanningConnector extends BaseFileScanningConnector { const s3Stream = (await getObjectStream(file.bucket, file.path)).Body as Readable s3Stream.pipe(avStream) log.info({ modelId: file.modelId, fileId: file._id, name: file.name }, 'Scan started.') - avStream.on('scan-complete', async (result) => { - log.info({ result, modelId: file.modelId, fileId: file._id, name: file.name }, 'Scan complete.') - updateFileWithResults( - file, - { - toolName: clamAvToolName, - state: ScanState.Complete, - isInfected: result.isInfected, - viruses: result.viruses, - }, - clamAvToolName, - ) - }) - avStream.on('error', async (error) => { - log.error({ error, modelId: file.modelId, fileId: file._id, name: file.name }, 'Scan errored.') - updateFileWithResults( - file, - { + const res: FileScanResult = await new Promise((resolve) => { + avStream + .on('scan-complete', (result) => { + log.info({ modelId: file.modelId, fileId: file._id, name: file.name, result }, 'Scan complete.') + resolve({ + toolName: clamAvToolName, + state: ScanState.Complete, + isInfected: result.isInfected, + viruses: result.viruses, + }) + }) + .on('error', (err) => { + log.error({ err, modelId: file.modelId, fileId: file._id, name: file.name }, 'Scan errored.') + resolve({ + toolName: clamAvToolName, + state: ScanState.Error, + }) + }) + avStream.on('timeout', async (error) => { + log.error({ error, modelId: file.modelId, fileId: file._id, name: file.name }, 'Scan timed out.') + resolve({ toolName: clamAvToolName, state: ScanState.Error, - }, - clamAvToolName, - ) - }) - avStream.on('timeout', async (error) => { - log.error({ error, modelId: file.modelId, fileId: file._id, name: file.name }, 'Scan timed out.') - updateFileWithResults( - file, - { - toolName: clamAvToolName, - state: ScanState.Error, - }, - clamAvToolName, - ) + }) + }) }) + return [res] } } diff --git a/backend/src/connectors/fileScanning/index.ts b/backend/src/connectors/fileScanning/index.ts index 542d407fd..0327f7ebd 100644 --- a/backend/src/connectors/fileScanning/index.ts +++ b/backend/src/connectors/fileScanning/index.ts @@ -1,9 +1,8 @@ -import { FileInterface, ScanState } from '../../models/File.js' -import { updateFileWithResults } from '../../services/file.js' import config from '../../utils/config.js' import { ConfigurationError } from '../../utils/error.js' import { BaseFileScanningConnector } from './Base.js' -import { ClamAvFileScanningConnector, clamAvToolName } from './clamAv.js' +import { ClamAvFileScanningConnector } from './clamAv.js' +import { FileScanningWrapper } from './wrapper.js' export const FileScanKind = { ClamAv: 'clamAV', @@ -11,12 +10,11 @@ export const FileScanKind = { export type FileScanKindKeys = (typeof FileScanKind)[keyof typeof FileScanKind] const fileScanConnectors: BaseFileScanningConnector[] = [] -export function getFileScanningConnectors(file: FileInterface) { +export default function runFileScanners() { config.connectors.fileScanners.kinds.forEach(async (fileScanner) => { switch (fileScanner) { case FileScanKind.ClamAv: fileScanConnectors.push(new ClamAvFileScanningConnector()) - await updateFileWithResults(file, { toolName: clamAvToolName, state: ScanState.InProgress }, clamAvToolName) break default: throw ConfigurationError(`'${fileScanner}' is not a valid file scanning kind.`, { @@ -24,5 +22,7 @@ export function getFileScanningConnectors(file: FileInterface) { }) } }) - return fileScanConnectors + + const wrapper = new FileScanningWrapper(fileScanConnectors) + return wrapper } diff --git a/backend/src/connectors/fileScanning/wrapper.ts b/backend/src/connectors/fileScanning/wrapper.ts new file mode 100644 index 000000000..0fc97549b --- /dev/null +++ b/backend/src/connectors/fileScanning/wrapper.ts @@ -0,0 +1,29 @@ +import { FileInterface } from '../../models/File.js' +import { BaseFileScanningConnector, FileScanResult } from './Base.js' + +export class FileScanningWrapper extends BaseFileScanningConnector { + scanners: BaseFileScanningConnector[] = [] + + constructor(scanners: BaseFileScanningConnector[]) { + super() + this.scanners = scanners + } + + info() { + const scannerNames: string[] = [] + for (const scanner of this.scanners) { + scannerNames.push(...scanner.info()) + } + return scannerNames + } + + async scan(file: FileInterface) { + const results: FileScanResult[] = [] + for (const scanner of this.scanners) { + const scannerResults = await scanner.scan(file) + results.push(...scannerResults) + } + + return results + } +} diff --git a/backend/src/routes.ts b/backend/src/routes.ts index 9f4d2f9ac..1eee41626 100644 --- a/backend/src/routes.ts +++ b/backend/src/routes.ts @@ -10,6 +10,7 @@ import { getDockerRegistryAuth } from './routes/v1/registryAuth.js' import { getCurrentUser } from './routes/v2/entities/getCurrentUser.js' import { getEntities } from './routes/v2/entities/getEntities.js' import { getEntityLookup } from './routes/v2/entities/getEntityLookup.js' +import { getFilescanningInfo } from './routes/v2/filescanning/getFilescanningInfo.js' import { deleteAccessRequest } from './routes/v2/model/accessRequest/deleteAccessRequest.js' import { getAccessRequest } from './routes/v2/model/accessRequest/getAccessRequest.js' import { getModelAccessRequests } from './routes/v2/model/accessRequest/getModelAccessRequests.js' @@ -210,6 +211,8 @@ server.delete('/api/v2/user/token/:accessKey', ...deleteUserToken) server.get('/api/v2/specification', ...getSpecification) +server.get('/api/v2/filescanning/info', ...getFilescanningInfo) + // Python docs const __filename = fileURLToPath(import.meta.url) const __dirname = path.dirname(__filename) diff --git a/backend/src/routes/v2/filescanning/getFilescanningInfo.ts b/backend/src/routes/v2/filescanning/getFilescanningInfo.ts new file mode 100644 index 000000000..4abc4bd71 --- /dev/null +++ b/backend/src/routes/v2/filescanning/getFilescanningInfo.ts @@ -0,0 +1,15 @@ +import bodyParser from 'body-parser' +import { Request, Response } from 'express' + +import runFileScanners from '../../../connectors/fileScanning/index.js' + +interface GetFileScanningInfoResponse { + scanners: string[] +} + +export const getFilescanningInfo = [ + bodyParser.json(), + async (req: Request, res: Response) => { + return res.json({ scanners: runFileScanners().info() }) + }, +] diff --git a/backend/src/services/file.ts b/backend/src/services/file.ts index dfd9b9b4d..db748b08f 100644 --- a/backend/src/services/file.ts +++ b/backend/src/services/file.ts @@ -4,8 +4,8 @@ import { getObjectStream, putObjectStream } from '../clients/s3.js' import { FileAction } from '../connectors/authorisation/actions.js' import authorisation from '../connectors/authorisation/index.js' import { FileScanResult } from '../connectors/fileScanning/Base.js' -import { getFileScanningConnectors } from '../connectors/fileScanning/index.js' -import FileModel, { FileInterface, ScanState } from '../models/File.js' +import runFileScanners from '../connectors/fileScanning/index.js' +import FileModel, { FileInterface } from '../models/File.js' import { UserInterface } from '../models/User.js' import config from '../utils/config.js' import { BadReq, Forbidden, NotFound } from '../utils/error.js' @@ -37,32 +37,33 @@ export async function uploadFile(user: UserInterface, modelId: string, name: str await file.save() if (config.ui.avScanning.enabled) { - const scanners = getFileScanningConnectors(file) - scanners.forEach((scanner) => { - scanner.scan(file) - }) + runFileScanners() + .scan(file) + .then((resultsArray) => updateFileWithResults(file, resultsArray)) } return file } -export async function updateFileWithResults(file: FileInterface, result: FileScanResult, toolName) { - if (!file.avScan.find((result) => result.toolName === toolName)) { - const updatedAvScanArray = [...file.avScan, { toolName: toolName, state: ScanState.InProgress }] - file.avScan = updatedAvScanArray - await FileModel.updateOne( - { _id: file._id }, - { - $set: { avScan: updatedAvScanArray }, - }, - ) - } else { - await FileModel.updateOne( - { _id: file._id, 'avScan.toolName': toolName }, - { - $set: { 'avScan.$': result }, - }, - ) +export async function updateFileWithResults(file: FileInterface, results: FileScanResult[]) { + for (const result of results) { + if (!file.avScan.find((avScanResult) => avScanResult.toolName === result.toolName)) { + const updatedAvScanArray = [...file.avScan, { toolName: result.toolName, state: result.state }] + file.avScan = updatedAvScanArray + await FileModel.updateOne( + { _id: file._id }, + { + $set: { avScan: updatedAvScanArray }, + }, + ) + } else { + await FileModel.updateOne( + { _id: file._id, 'avScan.toolName': result.toolName }, + { + $set: { 'avScan.$': result }, + }, + ) + } } } From fdfa0560219216d4ad35404aff946777c6457b78 Mon Sep 17 00:00:00 2001 From: araddcc002 Date: Thu, 10 Oct 2024 16:23:22 +0000 Subject: [PATCH 19/32] added frontend actions file for filescanning info call. updated ui to check for info insead of ui config for scanner list --- frontend/actions/fileScanning.ts | 20 +++++++++++++++++++ .../src/entry/model/releases/FileDownload.tsx | 12 +++++------ 2 files changed, 26 insertions(+), 6 deletions(-) create mode 100644 frontend/actions/fileScanning.ts diff --git a/frontend/actions/fileScanning.ts b/frontend/actions/fileScanning.ts new file mode 100644 index 000000000..8ec44e068 --- /dev/null +++ b/frontend/actions/fileScanning.ts @@ -0,0 +1,20 @@ +import useSWR from 'swr' +import { ErrorInfo, fetcher } from 'utils/fetcher' + +const emptyScannerList = [] + +export function useGetFileScannerInfo() { + const { data, isLoading, error, mutate } = useSWR< + { + scanners: string[] + }, + ErrorInfo + >('/api/v2/filescanning/info', fetcher) + + return { + scannersMutate: mutate, + scanners: data ? data : emptyScannerList, + isScannersLoading: isLoading, + isScannersError: error, + } +} diff --git a/frontend/src/entry/model/releases/FileDownload.tsx b/frontend/src/entry/model/releases/FileDownload.tsx index db959dec1..f45278ecf 100644 --- a/frontend/src/entry/model/releases/FileDownload.tsx +++ b/frontend/src/entry/model/releases/FileDownload.tsx @@ -1,6 +1,6 @@ import { Done, Error, Warning } from '@mui/icons-material' import { Chip, Divider, Grid, Link, Popover, Stack, Tooltip, Typography } from '@mui/material' -import { useGetUiConfig } from 'actions/uiConfig' +import { useGetFileScannerInfo } from 'actions/fileScanning' import prettyBytes from 'pretty-bytes' import { Fragment, useMemo, useState } from 'react' import Loading from 'src/common/Loading' @@ -16,7 +16,7 @@ type FileDownloadProps = { export default function FileDownload({ modelId, file }: FileDownloadProps) { const [anchorEl, setAnchorEl] = useState(null) - const { uiConfig, isUiConfigLoading, isUiConfigError } = useGetUiConfig() + const { scanners, isScannersLoading, isScannersError } = useGetFileScannerInfo() const open = Boolean(anchorEl) @@ -85,11 +85,11 @@ export default function FileDownload({ modelId, file }: FileDownloadProps) { ) }, [anchorEl, file, open]) - if (isUiConfigError) { - return + if (isScannersError) { + return } - if (isUiConfigLoading) { + if (isScannersLoading) { return } @@ -106,7 +106,7 @@ export default function FileDownload({ modelId, file }: FileDownloadProps) { - {uiConfig && uiConfig.avScanning && avChip} + {scanners && avChip} From 6dabbaeb72cef475bec6d7fb7db0afec50af0a2d Mon Sep 17 00:00:00 2001 From: araddcc002 Date: Mon, 14 Oct 2024 13:34:30 +0000 Subject: [PATCH 20/32] updated config and unit tests --- backend/config/default.cjs | 4 --- backend/config/docker_compose.cjs | 6 ---- backend/src/services/file.ts | 2 +- backend/src/services/mirroredModel.ts | 3 +- backend/src/types/types.ts | 4 --- backend/src/utils/__mocks__/config.ts | 3 -- backend/src/utils/config.ts | 4 --- backend/test/services/file.spec.ts | 28 ++++++++----------- backend/test/services/mirroredModel.spec.ts | 13 +++++++-- frontend/types/types.ts | 3 -- .../templates/bailo/bailo.configmap.yaml | 5 ---- infrastructure/helm/bailo/values.yaml | 3 -- 12 files changed, 25 insertions(+), 53 deletions(-) diff --git a/backend/config/default.cjs b/backend/config/default.cjs index 9dec4e023..102c84151 100644 --- a/backend/config/default.cjs +++ b/backend/config/default.cjs @@ -190,10 +190,6 @@ module.exports = { text: '', startTimestamp: '', }, - - avScanning: { - enabled: false, - }, }, connectors: { diff --git a/backend/config/docker_compose.cjs b/backend/config/docker_compose.cjs index 3b89c9181..940589d97 100644 --- a/backend/config/docker_compose.cjs +++ b/backend/config/docker_compose.cjs @@ -66,10 +66,4 @@ module.exports = { host: 'clamd', }, }, - - ui: { - avScanning: { - enabled: true, - }, - }, } diff --git a/backend/src/services/file.ts b/backend/src/services/file.ts index db748b08f..603553539 100644 --- a/backend/src/services/file.ts +++ b/backend/src/services/file.ts @@ -36,7 +36,7 @@ export async function uploadFile(user: UserInterface, modelId: string, name: str await file.save() - if (config.ui.avScanning.enabled) { + if (runFileScanners().info()) { runFileScanners() .scan(file) .then((resultsArray) => updateFileWithResults(file, resultsArray)) diff --git a/backend/src/services/mirroredModel.ts b/backend/src/services/mirroredModel.ts index 01f359e67..7b82fcac3 100644 --- a/backend/src/services/mirroredModel.ts +++ b/backend/src/services/mirroredModel.ts @@ -10,6 +10,7 @@ import { sign } from '../clients/kms.js' import { getObjectStream, putObjectStream } from '../clients/s3.js' import { ModelAction } from '../connectors/authorisation/actions.js' import authorisation from '../connectors/authorisation/index.js' +import runFileScanners from '../connectors/fileScanning/index.js' import { FileInterfaceDoc, ScanState } from '../models/File.js' import { ModelDoc } from '../models/Model.js' import { ModelCardRevisionInterface } from '../models/ModelCardRevision.js' @@ -341,7 +342,7 @@ async function checkReleaseFiles(user: UserInterface, modelId: string, semvers: } } - if (config.ui.avScanning.enabled) { + if (runFileScanners().info()) { const files: FileInterfaceDoc[] = await getFilesByIds(user, modelId, fileIds) const scanErrors: { missingScan: Array<{ name: string; id: string }> diff --git a/backend/src/types/types.ts b/backend/src/types/types.ts index ba4ff35b9..53e98700d 100644 --- a/backend/src/types/types.ts +++ b/backend/src/types/types.ts @@ -39,8 +39,4 @@ export interface UiConfig { text: string startTimestamp: string } - - avScanning: { - enabled: boolean - } } diff --git a/backend/src/utils/__mocks__/config.ts b/backend/src/utils/__mocks__/config.ts index 62ca6296b..b4e2deb08 100644 --- a/backend/src/utils/__mocks__/config.ts +++ b/backend/src/utils/__mocks__/config.ts @@ -35,9 +35,6 @@ const config = { instrumentation: { enabled: false, }, - avScanning: { - enabled: false, - }, } export default config diff --git a/backend/src/utils/config.ts b/backend/src/utils/config.ts index e5b4d47e9..1501c69a2 100644 --- a/backend/src/utils/config.ts +++ b/backend/src/utils/config.ts @@ -140,10 +140,6 @@ export interface Config { text: string startTimestamp: string } - - avScanning: { - enabled: boolean - } } session: { diff --git a/backend/test/services/file.spec.ts b/backend/test/services/file.spec.ts index 0804a0c43..90a16d82c 100644 --- a/backend/test/services/file.spec.ts +++ b/backend/test/services/file.spec.ts @@ -3,7 +3,7 @@ import { describe, expect, test, vi } from 'vitest' import { FileAction } from '../../src/connectors/authorisation/actions.js' import authorisation from '../../src/connectors/authorisation/index.js' -import { ClamAvFileScanningConnector } from '../../src/connectors/fileScanning/clamAv.js' +import { FileScanResult } from '../../src/connectors/fileScanning/Base.js' import { UserInterface } from '../../src/models/User.js' import { downloadFile, @@ -34,11 +34,6 @@ const configMock = vi.hoisted( port: 8080, }, }, - ui: { - avScanning: { - enabled: false, - }, - }, s3: { buckets: { uploads: 'uploads', @@ -57,10 +52,18 @@ vi.mock('../../src/utils/config.js', () => ({ default: configMock, })) -const fileScanningConnector = vi.hoisted(() => ({ - getFileScanningConnectors: vi.fn(() => [new ClamAvFileScanningConnector()]), +const fileScanResult: FileScanResult = { + state: 'complete', + isInfected: false, + toolName: 'Test', +} +const fileScanningMock = vi.hoisted(() => ({ + default: vi.fn(() => ({ + info: vi.fn(() => []), + scan: vi.fn(() => new Promise(() => fileScanResult)), + })), })) -vi.mock('../../src/connectors/fileScanning/index.js', () => fileScanningConnector) +vi.mock('../../src/connectors/fileScanning/index.js', async () => fileScanningMock) const s3Mocks = vi.hoisted(() => ({ putObjectStream: vi.fn(() => ({ fileSize: 100 })), @@ -68,11 +71,6 @@ const s3Mocks = vi.hoisted(() => ({ })) vi.mock('../../src/clients/s3.js', () => s3Mocks) -const clamAvScan = vi.hoisted(() => ({ - scan: vi.fn(() => {}), -})) -vi.mock('../../src/connectors/fileScanning/clamAv.js') - const modelMocks = vi.hoisted(() => ({ getModelById: vi.fn(() => ({ settings: { mirror: { sourceModelId: '' } } })), })) @@ -126,13 +124,11 @@ describe('services > file', () => { test('uploadFile > virus scan initialised', async () => { vi.spyOn(configMock, 'avScanning', 'get').mockReturnValue({ clamdscan: 'test' }) - vi.spyOn(configMock, 'ui', 'get').mockReturnValue({ avScanning: { enabled: true } }) vi.spyOn(configMock, 'connectors', 'get').mockReturnValue({ fileScanners: { kinds: ['clamAV'], }, }) - vi.mocked(clamAvScan.scan).mockImplementation(() => {}) const user = { dn: 'testUser' } as UserInterface const modelId = 'testModelId' const name = 'testFile' diff --git a/backend/test/services/mirroredModel.spec.ts b/backend/test/services/mirroredModel.spec.ts index 1a8a07201..27defa6d1 100644 --- a/backend/test/services/mirroredModel.spec.ts +++ b/backend/test/services/mirroredModel.spec.ts @@ -5,6 +5,13 @@ import authorisation from '../../src/connectors/authorisation/index.js' import { UserInterface } from '../../src/models/User.js' import { exportModel, importModel } from '../../src/services/mirroredModel.js' +const fileScanningMock = vi.hoisted(() => ({ + default: vi.fn(() => ({ + info: vi.fn(() => []), + })), +})) +vi.mock('../../src/connectors/fileScanning/index.js', async () => fileScanningMock) + const fflateMock = vi.hoisted(() => ({ unzipSync: vi.fn(), })) @@ -27,6 +34,9 @@ vi.mock('../../src/connectors/authorisation/index.js', async () => ({ default: authMock, })) +// const fileScanConnectors: BaseFileScanningConnector[] = [new ClamAvFileScanningConnector()] +// const wrapper = new FileScanningWrapper(fileScanConnectors) + const configMock = vi.hoisted( () => ({ @@ -34,9 +44,6 @@ const configMock = vi.hoisted( modelMirror: { enabled: true, }, - avScanning: { - enabled: true, - }, }, s3: { buckets: { uploads: 'test' } }, modelMirror: { diff --git a/frontend/types/types.ts b/frontend/types/types.ts index 1400f7a39..630b7cf9e 100644 --- a/frontend/types/types.ts +++ b/frontend/types/types.ts @@ -55,9 +55,6 @@ export interface UiConfig { text: string startTimestamp: string } - avScanning: { - enabled: boolean - } } export interface FileInterface { diff --git a/infrastructure/helm/bailo/templates/bailo/bailo.configmap.yaml b/infrastructure/helm/bailo/templates/bailo/bailo.configmap.yaml index 2d8e0ba9a..2d0fe34b6 100644 --- a/infrastructure/helm/bailo/templates/bailo/bailo.configmap.yaml +++ b/infrastructure/helm/bailo/templates/bailo/bailo.configmap.yaml @@ -63,7 +63,6 @@ data: }, avScanning: { - enabled: {{ .Values.clamav.enabled }}, clamdscan: { host: '{{ include "bailo.fullname" . }}-clamav', port: {{ .Values.clamav.port }}, @@ -175,10 +174,6 @@ data: enabled: {{ .Values.modelMirror.enabled }}, disclaimer: '{{ .Values.modelMirror.disclaimer }}' } - - avScanning: { - enabled: {{ .Values.config.ui.avScanning.enabled }}, - } }, connectors: { diff --git a/infrastructure/helm/bailo/values.yaml b/infrastructure/helm/bailo/values.yaml index 91786fc38..34df0d96c 100644 --- a/infrastructure/helm/bailo/values.yaml +++ b/infrastructure/helm/bailo/values.yaml @@ -230,9 +230,6 @@ config: text: '' startTimestamp: '' - avScanning: - enabled: false - smtp: #host: 'mail' #service name port: 1025 From c1c754835f9d9f4bd472fe4eb4139f5edda0609e Mon Sep 17 00:00:00 2001 From: araddcc002 Date: Mon, 14 Oct 2024 15:37:04 +0000 Subject: [PATCH 21/32] fixed a typo --- backend/src/connectors/fileScanning/clamAv.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/connectors/fileScanning/clamAv.ts b/backend/src/connectors/fileScanning/clamAv.ts index d4504af08..ab6d2c19f 100644 --- a/backend/src/connectors/fileScanning/clamAv.ts +++ b/backend/src/connectors/fileScanning/clamAv.ts @@ -34,7 +34,7 @@ export class ClamAvFileScanningConnector extends BaseFileScanningConnector { await this.init() if (!av) { throw ConfigurationError( - 'Clam AV does not look like it is running. Chec that it has been correctly initialised by calling the init function.', + 'Clam AV does not look like it is running. Check that it has been correctly initialised by calling the init function.', { clamAvConfig: config.avScanning, }, From f1ccde5f49633862f1df3b0c25a5889c46a80477 Mon Sep 17 00:00:00 2001 From: araddcc002 Date: Tue, 15 Oct 2024 16:06:39 +0000 Subject: [PATCH 22/32] various improvements including change to how we manage the initialisation of the file scanning --- backend/config/default.cjs | 2 +- backend/config/docker_compose.cjs | 6 ++++++ backend/src/connectors/fileScanning/clamAv.ts | 1 - backend/src/connectors/fileScanning/index.ts | 19 ++++++++++++----- .../v2/filescanning/getFilescanningInfo.ts | 3 ++- backend/src/services/file.ts | 21 ++++++++++++------- backend/src/services/mirroredModel.ts | 12 +++-------- 7 files changed, 39 insertions(+), 25 deletions(-) diff --git a/backend/config/default.cjs b/backend/config/default.cjs index 102c84151..6468af545 100644 --- a/backend/config/default.cjs +++ b/backend/config/default.cjs @@ -206,7 +206,7 @@ module.exports = { }, fileScanners: { - kinds: ['clamAV'], + kinds: [], }, }, diff --git a/backend/config/docker_compose.cjs b/backend/config/docker_compose.cjs index 940589d97..d113374d1 100644 --- a/backend/config/docker_compose.cjs +++ b/backend/config/docker_compose.cjs @@ -66,4 +66,10 @@ module.exports = { host: 'clamd', }, }, + + connectors: { + fileScanners: { + kinds: ['clamAV'], + }, + }, } diff --git a/backend/src/connectors/fileScanning/clamAv.ts b/backend/src/connectors/fileScanning/clamAv.ts index ab6d2c19f..8abc65daa 100644 --- a/backend/src/connectors/fileScanning/clamAv.ts +++ b/backend/src/connectors/fileScanning/clamAv.ts @@ -31,7 +31,6 @@ export class ClamAvFileScanningConnector extends BaseFileScanningConnector { } async scan(file: FileInterfaceDoc): Promise { - await this.init() if (!av) { throw ConfigurationError( 'Clam AV does not look like it is running. Check that it has been correctly initialised by calling the init function.', diff --git a/backend/src/connectors/fileScanning/index.ts b/backend/src/connectors/fileScanning/index.ts index 0327f7ebd..c74141099 100644 --- a/backend/src/connectors/fileScanning/index.ts +++ b/backend/src/connectors/fileScanning/index.ts @@ -10,11 +10,21 @@ export const FileScanKind = { export type FileScanKindKeys = (typeof FileScanKind)[keyof typeof FileScanKind] const fileScanConnectors: BaseFileScanningConnector[] = [] -export default function runFileScanners() { +let scannerWrapper: undefined | BaseFileScanningConnector = undefined +export default function runFileScanners(cache = true) { + if (scannerWrapper && cache) { + return scannerWrapper + } config.connectors.fileScanners.kinds.forEach(async (fileScanner) => { switch (fileScanner) { case FileScanKind.ClamAv: - fileScanConnectors.push(new ClamAvFileScanningConnector()) + try { + const scanner = new ClamAvFileScanningConnector() + scanner.init() + fileScanConnectors.push(scanner) + } catch (error) { + throw ConfigurationError('Could not configure or initialise Clam AV') + } break default: throw ConfigurationError(`'${fileScanner}' is not a valid file scanning kind.`, { @@ -22,7 +32,6 @@ export default function runFileScanners() { }) } }) - - const wrapper = new FileScanningWrapper(fileScanConnectors) - return wrapper + scannerWrapper = new FileScanningWrapper(fileScanConnectors) + return scannerWrapper } diff --git a/backend/src/routes/v2/filescanning/getFilescanningInfo.ts b/backend/src/routes/v2/filescanning/getFilescanningInfo.ts index 4abc4bd71..acf60a728 100644 --- a/backend/src/routes/v2/filescanning/getFilescanningInfo.ts +++ b/backend/src/routes/v2/filescanning/getFilescanningInfo.ts @@ -10,6 +10,7 @@ interface GetFileScanningInfoResponse { export const getFilescanningInfo = [ bodyParser.json(), async (req: Request, res: Response) => { - return res.json({ scanners: runFileScanners().info() }) + const scanners = runFileScanners() + return res.json({ scanners: scanners.info() }) }, ] diff --git a/backend/src/services/file.ts b/backend/src/services/file.ts index 603553539..0f3761476 100644 --- a/backend/src/services/file.ts +++ b/backend/src/services/file.ts @@ -5,7 +5,7 @@ import { FileAction } from '../connectors/authorisation/actions.js' import authorisation from '../connectors/authorisation/index.js' import { FileScanResult } from '../connectors/fileScanning/Base.js' import runFileScanners from '../connectors/fileScanning/index.js' -import FileModel, { FileInterface } from '../models/File.js' +import FileModel, { FileInterface, ScanState } from '../models/File.js' import { UserInterface } from '../models/User.js' import config from '../utils/config.js' import { BadReq, Forbidden, NotFound } from '../utils/error.js' @@ -36,10 +36,17 @@ export async function uploadFile(user: UserInterface, modelId: string, name: str await file.save() - if (runFileScanners().info()) { - runFileScanners() - .scan(file) - .then((resultsArray) => updateFileWithResults(file, resultsArray)) + const scanners = runFileScanners() + + if (scanners.info()) { + const resultsInprogress = scanners.info().map((scannerName) => { + return { + toolName: scannerName, + state: ScanState.InProgress, + } + }) + await updateFileWithResults(file, resultsInprogress) + scanners.scan(file).then((resultsArray) => updateFileWithResults(file, resultsArray)) } return file @@ -48,12 +55,10 @@ export async function uploadFile(user: UserInterface, modelId: string, name: str export async function updateFileWithResults(file: FileInterface, results: FileScanResult[]) { for (const result of results) { if (!file.avScan.find((avScanResult) => avScanResult.toolName === result.toolName)) { - const updatedAvScanArray = [...file.avScan, { toolName: result.toolName, state: result.state }] - file.avScan = updatedAvScanArray await FileModel.updateOne( { _id: file._id }, { - $set: { avScan: updatedAvScanArray }, + $set: { avScan: { toolName: result.toolName, state: result.state } }, }, ) } else { diff --git a/backend/src/services/mirroredModel.ts b/backend/src/services/mirroredModel.ts index 7b82fcac3..28763e7f0 100644 --- a/backend/src/services/mirroredModel.ts +++ b/backend/src/services/mirroredModel.ts @@ -342,7 +342,8 @@ async function checkReleaseFiles(user: UserInterface, modelId: string, semvers: } } - if (runFileScanners().info()) { + const scanners = runFileScanners() + if (scanners.info()) { const files: FileInterfaceDoc[] = await getFilesByIds(user, modelId, fileIds) const scanErrors: { missingScan: Array<{ name: string; id: string }> @@ -352,14 +353,7 @@ async function checkReleaseFiles(user: UserInterface, modelId: string, semvers: for (const file of files) { if (!file.avScan) { scanErrors.missingScan.push({ name: file.name, id: file.id }) - } else if ( - file.avScan.some( - (scanResult) => - scanResult.state === ScanState.NotScanned || - scanResult.state === ScanState.Error || - scanResult.state === ScanState.InProgress, - ) - ) { + } else if (file.avScan.some((scanResult) => scanResult.state !== ScanState.Complete)) { scanErrors.incompleteScan.push({ name: file.name, id: file.id }) } else if (file.avScan.some((scanResult) => scanResult.isInfected)) { scanErrors.failedScan.push({ name: file.name, id: file.id }) From bbdfe2b734332c2405d3ad2ae1a729d5631b69b4 Mon Sep 17 00:00:00 2001 From: araddcc002 Date: Wed, 16 Oct 2024 11:20:40 +0000 Subject: [PATCH 23/32] updated ui to better reflect av results --- backend/src/services/file.ts | 4 +- .../src/entry/model/releases/FileDownload.tsx | 52 +++++++++++++++---- 2 files changed, 44 insertions(+), 12 deletions(-) diff --git a/backend/src/services/file.ts b/backend/src/services/file.ts index 0f3761476..a90c92cba 100644 --- a/backend/src/services/file.ts +++ b/backend/src/services/file.ts @@ -46,7 +46,9 @@ export async function uploadFile(user: UserInterface, modelId: string, name: str } }) await updateFileWithResults(file, resultsInprogress) - scanners.scan(file).then((resultsArray) => updateFileWithResults(file, resultsArray)) + scanners.scan(file).then((resultsArray) => { + updateFileWithResults(file, resultsArray) + }) } return file diff --git a/frontend/src/entry/model/releases/FileDownload.tsx b/frontend/src/entry/model/releases/FileDownload.tsx index f45278ecf..578b13266 100644 --- a/frontend/src/entry/model/releases/FileDownload.tsx +++ b/frontend/src/entry/model/releases/FileDownload.tsx @@ -2,7 +2,7 @@ import { Done, Error, Warning } from '@mui/icons-material' import { Chip, Divider, Grid, Link, Popover, Stack, Tooltip, Typography } from '@mui/material' import { useGetFileScannerInfo } from 'actions/fileScanning' import prettyBytes from 'pretty-bytes' -import { Fragment, useMemo, useState } from 'react' +import { Fragment, ReactElement, useCallback, useMemo, useState } from 'react' import Loading from 'src/common/Loading' import MessageAlert from 'src/MessageAlert' import { FileInterface, isFileInterface, ScanState } from 'types/types' @@ -13,6 +13,12 @@ type FileDownloadProps = { file: FileInterface | File } +interface ChipDetails { + label: string + colour: 'error' | 'warning' | 'success' + icon: ReactElement +} + export default function FileDownload({ modelId, file }: FileDownloadProps) { const [anchorEl, setAnchorEl] = useState(null) @@ -20,6 +26,32 @@ export default function FileDownload({ modelId, file }: FileDownloadProps) { const open = Boolean(anchorEl) + const threatsFound = (file: FileInterface) => { + if (file.avScan === undefined) { + return 0 + } + return file.avScan.reduce((acc, scan) => { + return scan.viruses ? scan.viruses.length + acc : acc + }, 0) + } + + const chipDetails = useCallback((file: FileInterface): ChipDetails => { + if (file.avScan === undefined) { + return { label: 'Virus scan results could not be found.', colour: 'warning', icon: } + } + if (file.avScan.some((scan) => scan.state === ScanState.Error)) { + return { label: 'One or more virus scanning tools failed.', colour: 'warning', icon: } + } + if (threatsFound(file)) { + return { + label: `Virus scan failed: ${plural(threatsFound(file), 'threat')} found`, + colour: 'error', + icon: , + } + } + return { label: 'Virus scan passed', colour: 'success', icon: } + }, []) + const avChip = useMemo(() => { if ( !isFileInterface(file) || @@ -28,20 +60,17 @@ export default function FileDownload({ modelId, file }: FileDownloadProps) { ) { return } - const threatsFound = file.avScan.reduce((acc, scan) => { - return scan.viruses ? scan.viruses.length + acc : acc - }, 0) - if (file.avScan.some((scan) => scan.state !== ScanState.Complete)) { + if (file.avScan.some((scan) => scan.state === ScanState.InProgress)) { return } return ( <> : } + color={chipDetails(file).colour} + icon={chipDetails(file).icon} size='small' onClick={(e) => setAnchorEl(e.currentTarget)} - label={threatsFound ? `Virus scan failed: ${plural(threatsFound, 'threat')} found` : 'Virus scan passed'} + label={chipDetails(file).label} /> ) : ( - + {scanResult.state === 'error' ? : } - {scanResult.toolName} did not find any threats + {scanResult.toolName} + {scanResult.state === 'error' ? 'was not able to be run' : 'did not find any threats'} )} @@ -83,7 +113,7 @@ export default function FileDownload({ modelId, file }: FileDownloadProps) { ) - }, [anchorEl, file, open]) + }, [anchorEl, chipDetails, file, open]) if (isScannersError) { return From e9b6c42a33976f081bff38d9a4006c40c78c9218 Mon Sep 17 00:00:00 2001 From: araddcc002 Date: Wed, 16 Oct 2024 12:41:05 +0000 Subject: [PATCH 24/32] added backed log --- backend/src/connectors/fileScanning/clamAv.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/src/connectors/fileScanning/clamAv.ts b/backend/src/connectors/fileScanning/clamAv.ts index 8abc65daa..511edb5f6 100644 --- a/backend/src/connectors/fileScanning/clamAv.ts +++ b/backend/src/connectors/fileScanning/clamAv.ts @@ -23,6 +23,7 @@ export class ClamAvFileScanningConnector extends BaseFileScanningConnector { async init() { try { av = await new NodeClam().init({ clamdscan: config.avScanning.clamdscan }) + console.log(av) } catch (error) { throw ConfigurationError('Could not scan file as Clam AV is not running.', { clamAvConfig: config.avScanning, From 08b85d957b7794b5ca2fa68ad4a8129a655a559b Mon Sep 17 00:00:00 2001 From: araddcc002 Date: Wed, 16 Oct 2024 13:51:54 +0000 Subject: [PATCH 25/32] awaited clam av init call --- backend/src/connectors/fileScanning/clamAv.ts | 1 - backend/src/connectors/fileScanning/index.ts | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/src/connectors/fileScanning/clamAv.ts b/backend/src/connectors/fileScanning/clamAv.ts index 511edb5f6..8abc65daa 100644 --- a/backend/src/connectors/fileScanning/clamAv.ts +++ b/backend/src/connectors/fileScanning/clamAv.ts @@ -23,7 +23,6 @@ export class ClamAvFileScanningConnector extends BaseFileScanningConnector { async init() { try { av = await new NodeClam().init({ clamdscan: config.avScanning.clamdscan }) - console.log(av) } catch (error) { throw ConfigurationError('Could not scan file as Clam AV is not running.', { clamAvConfig: config.avScanning, diff --git a/backend/src/connectors/fileScanning/index.ts b/backend/src/connectors/fileScanning/index.ts index c74141099..b13bba462 100644 --- a/backend/src/connectors/fileScanning/index.ts +++ b/backend/src/connectors/fileScanning/index.ts @@ -20,7 +20,7 @@ export default function runFileScanners(cache = true) { case FileScanKind.ClamAv: try { const scanner = new ClamAvFileScanningConnector() - scanner.init() + await scanner.init() fileScanConnectors.push(scanner) } catch (error) { throw ConfigurationError('Could not configure or initialise Clam AV') From 5a18054f7ef9429a823df54f615faa1b237fb7b9 Mon Sep 17 00:00:00 2001 From: JR40159 <126243293+JR40159@users.noreply.github.com> Date: Tue, 22 Oct 2024 17:52:17 +0000 Subject: [PATCH 26/32] Update and simplify scanning process --- backend/config/docker_compose.cjs | 2 +- backend/src/connectors/fileScanning/clamAv.ts | 49 ++++++++----------- backend/src/connectors/fileScanning/index.ts | 4 +- .../src/connectors/fileScanning/wrapper.ts | 5 ++ .../v2/filescanning/getFilescanningInfo.ts | 3 +- backend/src/services/file.ts | 42 +++++++--------- backend/src/services/mirroredModel.ts | 3 +- docker-compose-prod.yml | 13 ++++- docker-compose.yml | 14 ++++-- 9 files changed, 72 insertions(+), 63 deletions(-) diff --git a/backend/config/docker_compose.cjs b/backend/config/docker_compose.cjs index d113374d1..497ffdfb6 100644 --- a/backend/config/docker_compose.cjs +++ b/backend/config/docker_compose.cjs @@ -69,7 +69,7 @@ module.exports = { connectors: { fileScanners: { - kinds: ['clamAV'], + kinds: [], }, }, } diff --git a/backend/src/connectors/fileScanning/clamAv.ts b/backend/src/connectors/fileScanning/clamAv.ts index 8abc65daa..0173384d3 100644 --- a/backend/src/connectors/fileScanning/clamAv.ts +++ b/backend/src/connectors/fileScanning/clamAv.ts @@ -39,36 +39,29 @@ export class ClamAvFileScanningConnector extends BaseFileScanningConnector { }, ) } - const avStream = av.passthrough() const s3Stream = (await getObjectStream(file.bucket, file.path)).Body as Readable - s3Stream.pipe(avStream) - log.info({ modelId: file.modelId, fileId: file._id, name: file.name }, 'Scan started.') - const res: FileScanResult = await new Promise((resolve) => { - avStream - .on('scan-complete', (result) => { - log.info({ modelId: file.modelId, fileId: file._id, name: file.name, result }, 'Scan complete.') - resolve({ - toolName: clamAvToolName, - state: ScanState.Complete, - isInfected: result.isInfected, - viruses: result.viruses, - }) - }) - .on('error', (err) => { - log.error({ err, modelId: file.modelId, fileId: file._id, name: file.name }, 'Scan errored.') - resolve({ - toolName: clamAvToolName, - state: ScanState.Error, - }) - }) - avStream.on('timeout', async (error) => { - log.error({ error, modelId: file.modelId, fileId: file._id, name: file.name }, 'Scan timed out.') - resolve({ + try { + const { isInfected, viruses } = await av.scanStream(s3Stream) + log.info( + { modelId: file.modelId, fileId: file._id, name: file.name, result: { isInfected, viruses } }, + 'Scan complete.', + ) + return [ + { + toolName: clamAvToolName, + state: ScanState.Complete, + isInfected, + viruses, + }, + ] + } catch (error) { + log.error({ error, modelId: file.modelId, fileId: file._id, name: file.name }, 'Scan errored.') + return [ + { toolName: clamAvToolName, state: ScanState.Error, - }) - }) - }) - return [res] + }, + ] + } } } diff --git a/backend/src/connectors/fileScanning/index.ts b/backend/src/connectors/fileScanning/index.ts index b13bba462..d005c4c38 100644 --- a/backend/src/connectors/fileScanning/index.ts +++ b/backend/src/connectors/fileScanning/index.ts @@ -11,7 +11,7 @@ export type FileScanKindKeys = (typeof FileScanKind)[keyof typeof FileScanKind] const fileScanConnectors: BaseFileScanningConnector[] = [] let scannerWrapper: undefined | BaseFileScanningConnector = undefined -export default function runFileScanners(cache = true) { +export function runFileScanners(cache = true) { if (scannerWrapper && cache) { return scannerWrapper } @@ -35,3 +35,5 @@ export default function runFileScanners(cache = true) { scannerWrapper = new FileScanningWrapper(fileScanConnectors) return scannerWrapper } + +export default runFileScanners() diff --git a/backend/src/connectors/fileScanning/wrapper.ts b/backend/src/connectors/fileScanning/wrapper.ts index 0fc97549b..d85bd1642 100644 --- a/backend/src/connectors/fileScanning/wrapper.ts +++ b/backend/src/connectors/fileScanning/wrapper.ts @@ -1,4 +1,5 @@ import { FileInterface } from '../../models/File.js' +import log from '../../services/log.js' import { BaseFileScanningConnector, FileScanResult } from './Base.js' export class FileScanningWrapper extends BaseFileScanningConnector { @@ -20,6 +21,10 @@ export class FileScanningWrapper extends BaseFileScanningConnector { async scan(file: FileInterface) { const results: FileScanResult[] = [] for (const scanner of this.scanners) { + log.info( + { modelId: file.modelId, fileId: file._id, name: file.name, toolName: scanner.info().pop() }, + 'Scan started.', + ) const scannerResults = await scanner.scan(file) results.push(...scannerResults) } diff --git a/backend/src/routes/v2/filescanning/getFilescanningInfo.ts b/backend/src/routes/v2/filescanning/getFilescanningInfo.ts index acf60a728..87aadec45 100644 --- a/backend/src/routes/v2/filescanning/getFilescanningInfo.ts +++ b/backend/src/routes/v2/filescanning/getFilescanningInfo.ts @@ -1,7 +1,7 @@ import bodyParser from 'body-parser' import { Request, Response } from 'express' -import runFileScanners from '../../../connectors/fileScanning/index.js' +import scanners from '../../../connectors/fileScanning/index.js' interface GetFileScanningInfoResponse { scanners: string[] @@ -10,7 +10,6 @@ interface GetFileScanningInfoResponse { export const getFilescanningInfo = [ bodyParser.json(), async (req: Request, res: Response) => { - const scanners = runFileScanners() return res.json({ scanners: scanners.info() }) }, ] diff --git a/backend/src/services/file.ts b/backend/src/services/file.ts index a90c92cba..cfdd9c83f 100644 --- a/backend/src/services/file.ts +++ b/backend/src/services/file.ts @@ -1,11 +1,12 @@ +import { Schema } from 'mongoose' import { Readable } from 'stream' import { getObjectStream, putObjectStream } from '../clients/s3.js' import { FileAction } from '../connectors/authorisation/actions.js' import authorisation from '../connectors/authorisation/index.js' import { FileScanResult } from '../connectors/fileScanning/Base.js' -import runFileScanners from '../connectors/fileScanning/index.js' -import FileModel, { FileInterface, ScanState } from '../models/File.js' +import scanners from '../connectors/fileScanning/index.js' +import FileModel, { ScanState } from '../models/File.js' import { UserInterface } from '../models/User.js' import config from '../utils/config.js' import { BadReq, Forbidden, NotFound } from '../utils/error.js' @@ -36,40 +37,33 @@ export async function uploadFile(user: UserInterface, modelId: string, name: str await file.save() - const scanners = runFileScanners() - if (scanners.info()) { - const resultsInprogress = scanners.info().map((scannerName) => { - return { - toolName: scannerName, - state: ScanState.InProgress, - } - }) - await updateFileWithResults(file, resultsInprogress) - scanners.scan(file).then((resultsArray) => { - updateFileWithResults(file, resultsArray) - }) + const resultsInprogress = scanners.info().map((scannerName) => ({ + toolName: scannerName, + state: ScanState.InProgress, + })) + await updateFileWithResults(file._id, resultsInprogress) + scanners.scan(file).then((resultsArray) => updateFileWithResults(file._id, resultsArray)) } return file } -export async function updateFileWithResults(file: FileInterface, results: FileScanResult[]) { +async function updateFileWithResults(_id: Schema.Types.ObjectId, results: FileScanResult[]) { for (const result of results) { - if (!file.avScan.find((avScanResult) => avScanResult.toolName === result.toolName)) { + const updateExistingResult = await FileModel.updateOne( + { _id, 'avScan.toolName': result.toolName }, + { + $set: { 'avScan.$': result }, + }, + ) + if (updateExistingResult.modifiedCount === 0) { await FileModel.updateOne( - { _id: file._id }, + { _id }, { $set: { avScan: { toolName: result.toolName, state: result.state } }, }, ) - } else { - await FileModel.updateOne( - { _id: file._id, 'avScan.toolName': result.toolName }, - { - $set: { 'avScan.$': result }, - }, - ) } } } diff --git a/backend/src/services/mirroredModel.ts b/backend/src/services/mirroredModel.ts index ebdc35e93..c590faff6 100644 --- a/backend/src/services/mirroredModel.ts +++ b/backend/src/services/mirroredModel.ts @@ -10,7 +10,7 @@ import { sign } from '../clients/kms.js' import { getObjectStream, putObjectStream } from '../clients/s3.js' import { ModelAction } from '../connectors/authorisation/actions.js' import authorisation from '../connectors/authorisation/index.js' -import runFileScanners from '../connectors/fileScanning/index.js' +import scanners from '../connectors/fileScanning/index.js' import { FileInterfaceDoc, ScanState } from '../models/File.js' import { ModelDoc } from '../models/Model.js' import { ModelCardRevisionInterface } from '../models/ModelCardRevision.js' @@ -355,7 +355,6 @@ async function checkReleaseFiles(user: UserInterface, modelId: string, semvers: } } - const scanners = runFileScanners() if (scanners.info()) { const files: FileInterfaceDoc[] = await getFilesByIds(user, modelId, fileIds) const scanErrors: { diff --git a/docker-compose-prod.yml b/docker-compose-prod.yml index 5aa68c551..303086bf2 100644 --- a/docker-compose-prod.yml +++ b/docker-compose-prod.yml @@ -27,6 +27,11 @@ services: clamd: image: clamav/clamav:1.3 + healthcheck: + test: ['CMD-SHELL', "echo 'PING' | nc -w 5 localhost 3310"] + interval: 30s + timeout: 10s + retries: 5 mailcrab: image: marlonb/mailcrab:v1.2.0 @@ -87,8 +92,12 @@ services: - ./backend/certs:/usr/local/share/ca-certificates - ./backend/config:/app/config depends_on: - - mongo - - minio + clamd: + condition: service_healthy + minio: + condition: service_started + mongo: + condition: service_started security_opt: - seccomp:unconfined - apparmor:unconfined diff --git a/docker-compose.yml b/docker-compose.yml index 68a1a788d..cc88d334f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,6 +29,11 @@ services: clamd: image: clamav/clamav:1.3 + healthcheck: + test: ['CMD-SHELL', "echo 'PING' | nc -w 5 localhost 3310"] + interval: 30s + timeout: 10s + retries: 5 mailcrab: image: marlonb/mailcrab:v1.2.0 @@ -92,9 +97,12 @@ services: - 3001:3001 - 9229:9229 depends_on: - - mongo - - minio - - clamd + clamd: + condition: service_healthy + minio: + condition: service_started + mongo: + condition: service_started security_opt: - seccomp:unconfined - apparmor:unconfined From ab64bd1ed8fa83c6857e78a7f58206160fdc513c Mon Sep 17 00:00:00 2001 From: JR40159 <126243293+JR40159@users.noreply.github.com> Date: Tue, 22 Oct 2024 17:55:31 +0000 Subject: [PATCH 27/32] Fix config that was removed by mistake --- backend/config/docker_compose.cjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/config/docker_compose.cjs b/backend/config/docker_compose.cjs index 497ffdfb6..d113374d1 100644 --- a/backend/config/docker_compose.cjs +++ b/backend/config/docker_compose.cjs @@ -69,7 +69,7 @@ module.exports = { connectors: { fileScanners: { - kinds: [], + kinds: ['clamAV'], }, }, } From 32e743d2ed846adffe27ff5686d7a44dc751ae58 Mon Sep 17 00:00:00 2001 From: araddcc002 Date: Tue, 29 Oct 2024 16:19:40 +0000 Subject: [PATCH 28/32] removed uneeded config mocking --- backend/config/test.cjs | 13 ++++- backend/src/utils/__mocks__/config.ts | 7 ++- .../authorisation/authorisation.spec.ts | 51 +++++++++++++++++-- .../test/routes/entities/getEntities.spec.ts | 1 - .../routes/entities/getEntityLookup.spec.ts | 2 - .../middleware/expressErrorHandler.spec.ts | 1 - .../patchAccessRequestComment.spec.ts.snap | 15 ------ .../accessRequest/deleteAccessRequest.spec.ts | 1 - .../accessRequest/getAccessRequest.spec.ts | 1 - ...ccessRequestCurrentUserPermissions.spec.ts | 1 - .../getModelAccessRequests.spec.ts | 1 - .../accessRequest/patchAccessRequest.spec.ts | 1 - .../accessRequest/postAccessRequest.spec.ts | 15 +++++- .../postAccessRequestComment.spec.ts | 1 - .../test/routes/model/file/deleteFile.spec.ts | 2 - .../test/routes/model/file/getFiles.spec.ts | 2 - .../model/file/postSimpleUpload.spec.ts | 2 - backend/test/routes/model/getModel.spec.ts | 2 - .../getModelCurrentUserPermissions.spec.ts | 2 - .../test/routes/model/getModelsSearch.spec.ts | 2 - .../routes/model/images/getImages.spec.ts | 2 - .../createInferencingService.spec.ts | 3 +- .../inferencing/getInferencingService.spec.ts | 1 - .../getInferencingServices.spec.ts | 1 - .../updateInferencingService.spec.ts | 1 - .../model/modelcard/getModelCard.spec.ts | 2 - .../model/modelcard/getModelCardHtml.spec.ts | 2 - .../modelcard/getModelCardRevisions.spec.ts | 2 - .../model/modelcard/postFromSchema.spec.ts | 2 - .../model/modelcard/postFromTemplate.spec.ts | 2 - .../model/modelcard/putModelCard.spec.ts | 2 - backend/test/routes/model/patchModel.spec.ts | 2 - backend/test/routes/model/postModel.spec.ts | 2 - .../routes/model/postRequestExport.spec.ts | 2 - .../routes/model/postRequestImport.spec.ts | 2 - .../model/release/deleteRelease.spec.ts | 2 - .../routes/model/release/getRelease.spec.ts | 2 - .../routes/model/release/getReleases.spec.ts | 2 - .../routes/model/release/postRelease.spec.ts | 2 - .../model/release/postReleaseComment.spec.ts | 2 - .../routes/model/release/putRelease.spec.ts | 2 - .../model/webhook/deleteWebhook.spec.ts | 2 - .../routes/model/webhook/getWebhooks.spec.ts | 2 - .../routes/model/webhook/postWebhook.spec.ts | 2 - .../routes/model/webhook/putWebhook.spec.ts | 2 - backend/test/routes/review/getReviews.spec.ts | 2 - .../postAccessRequestReviewResponse.spec.ts | 2 - .../review/postReleaseReviewResponse.spec.ts | 2 - .../test/routes/schema/deleteSchema.spec.ts | 2 - backend/test/routes/schema/getSchema.spec.ts | 2 - backend/test/routes/schema/getSchemas.spec.ts | 2 - .../test/routes/schema/patchSchema.spec.ts | 2 - backend/test/routes/schema/postSchema.spec.ts | 2 - backend/test/services/file.spec.ts | 10 ++-- backend/test/services/mirroredModel.spec.ts | 17 ++++--- 55 files changed, 95 insertions(+), 119 deletions(-) delete mode 100644 backend/test/routes/model/accessRequest/__snapshots__/patchAccessRequestComment.spec.ts.snap diff --git a/backend/config/test.cjs b/backend/config/test.cjs index 4ba52ba2c..688cfa41b 100644 --- a/backend/config/test.cjs +++ b/backend/config/test.cjs @@ -1 +1,12 @@ -module.exports = {} +module.exports = { + ui: { + inference: { + enabled: true, + }, + }, + connectors: { + fileScanners: { + kinds: [], + }, + }, +} diff --git a/backend/src/utils/__mocks__/config.ts b/backend/src/utils/__mocks__/config.ts index b4e2deb08..1d8daf572 100644 --- a/backend/src/utils/__mocks__/config.ts +++ b/backend/src/utils/__mocks__/config.ts @@ -21,7 +21,7 @@ const config = { kind: 'basic', }, fileScanners: { - kinds: ['clamAV'], + kinds: [], }, }, log: { @@ -35,6 +35,11 @@ const config = { instrumentation: { enabled: false, }, + ui: { + inference: { + enabled: true, + }, + }, } export default config diff --git a/backend/test/connectors/authorisation/authorisation.spec.ts b/backend/test/connectors/authorisation/authorisation.spec.ts index 184607ec8..3c492ebf3 100644 --- a/backend/test/connectors/authorisation/authorisation.spec.ts +++ b/backend/test/connectors/authorisation/authorisation.spec.ts @@ -1,9 +1,10 @@ import { describe, expect, test, vi } from 'vitest' import { getAuthorisationConnector } from '../../../src/connectors/authorisation/index.js' -import config from '../../../src/utils/__mocks__/config.js' +import { FileScanResult } from '../../../src/connectors/fileScanning/Base.js' +// import config from '../../../src/utils/__mocks__/config.js' -vi.mock('../../../src/utils/config.js') +// vi.mock('../../../src/utils/config.js') vi.mock('../../../src/connectors/authentication/index.js', () => ({ default: { @@ -11,6 +12,50 @@ vi.mock('../../../src/connectors/authentication/index.js', () => ({ }, })) +const fileScanResult: FileScanResult = { + state: 'complete', + isInfected: false, + toolName: 'Test', +} + +const fileScanningMock = vi.hoisted(() => ({ + info: vi.fn(() => []), + scan: vi.fn(() => new Promise(() => [fileScanResult])), +})) +vi.mock('../../src/connectors/fileScanning/index.js', async () => ({ default: fileScanningMock })) + +const configMock = vi.hoisted(() => ({ + app: { + protocol: 'http', + }, + instrumentation: { + enabled: false, + }, + log: { + level: 'info', + }, + connectors: { + authorisation: { + kind: 'basic', + }, + audit: { + kind: 'silly', + }, + fileScanners: { + kinds: [], + }, + }, + registry: { + connection: { + internal: 'https://localhost:5000', + }, + }, +})) +vi.mock('../../../src/utils/config.js', () => ({ + __esModule: true, + default: configMock, +})) + describe('connectors > authorisation', () => { test('basic', () => { const connector = getAuthorisationConnector(false) @@ -19,7 +64,7 @@ describe('connectors > authorisation', () => { test('invalid', () => { const invalidConnector = 'invalid' - config.connectors.authorisation.kind = invalidConnector + configMock.connectors.authorisation.kind = invalidConnector expect(() => getAuthorisationConnector(false)).toThrowError( `'${invalidConnector}' is not a valid authorisation kind.`, diff --git a/backend/test/routes/entities/getEntities.spec.ts b/backend/test/routes/entities/getEntities.spec.ts index 80d13e75e..040bf567d 100644 --- a/backend/test/routes/entities/getEntities.spec.ts +++ b/backend/test/routes/entities/getEntities.spec.ts @@ -3,7 +3,6 @@ import { describe, expect, test, vi } from 'vitest' import { testGet } from '../../testUtils/routes.js' vi.mock('../../../src/utils/user.js') -vi.mock('../../../src/utils/config.js') const authenticationMocks = vi.hoisted(() => ({ authenticationMiddleware: vi.fn(() => []), diff --git a/backend/test/routes/entities/getEntityLookup.spec.ts b/backend/test/routes/entities/getEntityLookup.spec.ts index 34a5639ae..b99bd7d08 100644 --- a/backend/test/routes/entities/getEntityLookup.spec.ts +++ b/backend/test/routes/entities/getEntityLookup.spec.ts @@ -2,9 +2,7 @@ import { describe, expect, test, vi } from 'vitest' import { testGet } from '../../testUtils/routes.js' -vi.mock('../../../src/utils/config.js') vi.mock('../../../src/utils/user.js') -vi.mock('../../../src/utils/config.js') const authenticationMocks = vi.hoisted(() => ({ authenticationMiddleware: vi.fn(() => []), diff --git a/backend/test/routes/middleware/expressErrorHandler.spec.ts b/backend/test/routes/middleware/expressErrorHandler.spec.ts index ff4ebfcf3..dc21529d1 100644 --- a/backend/test/routes/middleware/expressErrorHandler.spec.ts +++ b/backend/test/routes/middleware/expressErrorHandler.spec.ts @@ -3,7 +3,6 @@ import { describe, expect, test, vi } from 'vitest' import audit from '../../../src/connectors/audit/__mocks__/index.js' import { bailoErrorGuard, expressErrorHandler } from '../../../src/routes/middleware/expressErrorHandler.js' -vi.mock('../../../src/utils/config.js') vi.mock('../../../src/connectors/audit/index.js') describe('middleware > expressErrorHandler', () => { diff --git a/backend/test/routes/model/accessRequest/__snapshots__/patchAccessRequestComment.spec.ts.snap b/backend/test/routes/model/accessRequest/__snapshots__/patchAccessRequestComment.spec.ts.snap deleted file mode 100644 index 2c8faf897..000000000 --- a/backend/test/routes/model/accessRequest/__snapshots__/patchAccessRequestComment.spec.ts.snap +++ /dev/null @@ -1,15 +0,0 @@ -// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html - -exports[`routes > release > patchReleaseComment > 200 > ok 1`] = ` -{ - "accessRequest": { - "message": "test", - }, -} -`; - -exports[`routes > release > patchReleaseComment > audit > expected call 1`] = ` -{ - "message": "test", -} -`; diff --git a/backend/test/routes/model/accessRequest/deleteAccessRequest.spec.ts b/backend/test/routes/model/accessRequest/deleteAccessRequest.spec.ts index 2fb81f41f..4e27ebb6e 100644 --- a/backend/test/routes/model/accessRequest/deleteAccessRequest.spec.ts +++ b/backend/test/routes/model/accessRequest/deleteAccessRequest.spec.ts @@ -5,7 +5,6 @@ import { deleteAccessRequestSchema } from '../../../../src/routes/v2/model/acces import { createFixture, testDelete } from '../../../testUtils/routes.js' vi.mock('../../../../src/utils/user.js') -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/connectors/audit/index.js') vi.mock('../../../../src/connectors/authorisation/index.js') diff --git a/backend/test/routes/model/accessRequest/getAccessRequest.spec.ts b/backend/test/routes/model/accessRequest/getAccessRequest.spec.ts index 6950fbd8c..3c000674a 100644 --- a/backend/test/routes/model/accessRequest/getAccessRequest.spec.ts +++ b/backend/test/routes/model/accessRequest/getAccessRequest.spec.ts @@ -7,7 +7,6 @@ import { createFixture, testGet } from '../../../testUtils/routes.js' import { testAccessRequest } from '../../../testUtils/testModels.js' vi.mock('../../../../src/utils/user.js') -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/connectors/audit/index.js') vi.mock('../../../../src/connectors/authorisation/index.js') diff --git a/backend/test/routes/model/accessRequest/getAccessRequestCurrentUserPermissions.spec.ts b/backend/test/routes/model/accessRequest/getAccessRequestCurrentUserPermissions.spec.ts index f8e8cd352..83fe636c3 100644 --- a/backend/test/routes/model/accessRequest/getAccessRequestCurrentUserPermissions.spec.ts +++ b/backend/test/routes/model/accessRequest/getAccessRequestCurrentUserPermissions.spec.ts @@ -4,7 +4,6 @@ import { getAccessRequestCurrentUserPermissionsSchema } from '../../../../src/ro import { createFixture, testGet } from '../../../testUtils/routes.js' vi.mock('../../../../src/utils/user.js') -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/connectors/audit/index.js') vi.mock('../../../../src/connectors/authorisation/index.js') diff --git a/backend/test/routes/model/accessRequest/getModelAccessRequests.spec.ts b/backend/test/routes/model/accessRequest/getModelAccessRequests.spec.ts index 0253edd14..1a8d93f7a 100644 --- a/backend/test/routes/model/accessRequest/getModelAccessRequests.spec.ts +++ b/backend/test/routes/model/accessRequest/getModelAccessRequests.spec.ts @@ -4,7 +4,6 @@ import audit from '../../../../src/connectors/audit/__mocks__/index.js' import { getModelAccessRequestsSchema } from '../../../../src/routes/v2/model/accessRequest/getModelAccessRequests.js' import { createFixture, testGet } from '../../../testUtils/routes.js' -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/utils/user.js') vi.mock('../../../../src/connectors/audit/index.js') vi.mock('../../../../src/connectors/authorisation/index.js') diff --git a/backend/test/routes/model/accessRequest/patchAccessRequest.spec.ts b/backend/test/routes/model/accessRequest/patchAccessRequest.spec.ts index 28b1f5aa8..4c8aa1a81 100644 --- a/backend/test/routes/model/accessRequest/patchAccessRequest.spec.ts +++ b/backend/test/routes/model/accessRequest/patchAccessRequest.spec.ts @@ -5,7 +5,6 @@ import { patchAccessRequestSchema } from '../../../../src/routes/v2/model/access import { createFixture, testPatch } from '../../../testUtils/routes.js' vi.mock('../../../../src/utils/user.js') -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/connectors/audit/index.js') vi.mock('../../../../src/connectors/authorisation/index.js') diff --git a/backend/test/routes/model/accessRequest/postAccessRequest.spec.ts b/backend/test/routes/model/accessRequest/postAccessRequest.spec.ts index 234228ccb..c0bf1c810 100644 --- a/backend/test/routes/model/accessRequest/postAccessRequest.spec.ts +++ b/backend/test/routes/model/accessRequest/postAccessRequest.spec.ts @@ -1,10 +1,10 @@ import { describe, expect, test, vi } from 'vitest' import audit from '../../../../src/connectors/audit/__mocks__/index.js' +import { FileScanResult } from '../../../../src/connectors/fileScanning/Base.js' import { postAccessRequestSchema } from '../../../../src/routes/v2/model/accessRequest/postAccessRequest.js' import { createFixture, testPost } from '../../../testUtils/routes.js' -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/utils/user.js') vi.mock('../../../../src/connectors/audit/index.js') vi.mock('../../../../src/connectors/authorisation/index.js') @@ -13,6 +13,19 @@ vi.mock('../../../../src/services/accessRequest.js', async () => ({ createAccessRequest: vi.fn(() => ({ _id: 'test' })), })) +const fileScanResult: FileScanResult = { + state: 'complete', + isInfected: false, + toolName: 'Test', +} + +const fileScanningMock = vi.hoisted(() => ({ + info: vi.fn(() => []), + scan: vi.fn(() => new Promise(() => [fileScanResult])), + init: vi.fn(() => {}), +})) +vi.mock('../../src/connectors/fileScanning/index.js', async () => ({ default: fileScanningMock })) + describe('routes > accessRequest > postAccessRequest', () => { test('200 > ok', async () => { const fixture = createFixture(postAccessRequestSchema) diff --git a/backend/test/routes/model/accessRequest/postAccessRequestComment.spec.ts b/backend/test/routes/model/accessRequest/postAccessRequestComment.spec.ts index eb65e3e39..50b16e4c4 100644 --- a/backend/test/routes/model/accessRequest/postAccessRequestComment.spec.ts +++ b/backend/test/routes/model/accessRequest/postAccessRequestComment.spec.ts @@ -4,7 +4,6 @@ import audit from '../../../../src/connectors/audit/__mocks__/index.js' import { postAccessRequestCommentSchema } from '../../../../src/routes/v2/model/accessRequest/postAccessRequestComment.js' import { createFixture, testPost } from '../../../testUtils/routes.js' -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/utils/user.js') vi.mock('../../../../src/connectors/audit/index.js') vi.mock('../../../../src/connectors/authorisation/index.js') diff --git a/backend/test/routes/model/file/deleteFile.spec.ts b/backend/test/routes/model/file/deleteFile.spec.ts index 7f1224d07..bb1680b64 100644 --- a/backend/test/routes/model/file/deleteFile.spec.ts +++ b/backend/test/routes/model/file/deleteFile.spec.ts @@ -4,9 +4,7 @@ import audit from '../../../../src/connectors/audit/__mocks__/index.js' import { deleteFileSchema } from '../../../../src/routes/v2/model/file/deleteFile.js' import { createFixture, testDelete } from '../../../testUtils/routes.js' -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/utils/user.js') -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/connectors/audit/index.js') vi.mock('../../../../src/connectors/authorisation/index.js') diff --git a/backend/test/routes/model/file/getFiles.spec.ts b/backend/test/routes/model/file/getFiles.spec.ts index ecdbef4c4..ba64887e7 100644 --- a/backend/test/routes/model/file/getFiles.spec.ts +++ b/backend/test/routes/model/file/getFiles.spec.ts @@ -4,9 +4,7 @@ import audit from '../../../../src/connectors/audit/__mocks__/index.js' import { getFilesSchema } from '../../../../src/routes/v2/model/file/getFiles.js' import { createFixture, testGet } from '../../../testUtils/routes.js' -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/utils/user.js') -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/connectors/audit/index.js') vi.mock('../../../../src/connectors/authorisation/index.js') diff --git a/backend/test/routes/model/file/postSimpleUpload.spec.ts b/backend/test/routes/model/file/postSimpleUpload.spec.ts index 35e7823e4..0c8b18e1e 100644 --- a/backend/test/routes/model/file/postSimpleUpload.spec.ts +++ b/backend/test/routes/model/file/postSimpleUpload.spec.ts @@ -5,9 +5,7 @@ import audit from '../../../../src/connectors/audit/__mocks__/index.js' import { postSimpleUploadSchema } from '../../../../src/routes/v2/model/file/postSimpleUpload.js' import { createFixture, testPost } from '../../../testUtils/routes.js' -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/utils/user.js') -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/connectors/audit/index.js') vi.mock('../../../../src/connectors/authorisation/index.js') diff --git a/backend/test/routes/model/getModel.spec.ts b/backend/test/routes/model/getModel.spec.ts index 881a8ef53..26007ce70 100644 --- a/backend/test/routes/model/getModel.spec.ts +++ b/backend/test/routes/model/getModel.spec.ts @@ -4,9 +4,7 @@ import audit from '../../../src/connectors/audit/__mocks__/index.js' import { getModelSchema } from '../../../src/routes/v2/model/getModel.js' import { createFixture, testGet } from '../../testUtils/routes.js' -vi.mock('../../../src/utils/config.js') vi.mock('../../../src/utils/user.js') -vi.mock('../../../src/utils/config.js') vi.mock('../../../src/connectors/audit/index.js') describe('routes > model > getModel', () => { diff --git a/backend/test/routes/model/getModelCurrentUserPermissions.spec.ts b/backend/test/routes/model/getModelCurrentUserPermissions.spec.ts index d69263cea..ec34e402e 100644 --- a/backend/test/routes/model/getModelCurrentUserPermissions.spec.ts +++ b/backend/test/routes/model/getModelCurrentUserPermissions.spec.ts @@ -3,9 +3,7 @@ import { describe, expect, test, vi } from 'vitest' import { getModelCurrentUserPermissionsSchema } from '../../../src/routes/v2/model/getModelCurrentUserPermissions.js' import { createFixture, testGet } from '../../testUtils/routes.js' -vi.mock('../../../src/utils/config.js') vi.mock('../../../src/utils/user.js') -vi.mock('../../../src/utils/config.js') vi.mock('../../../src/connectors/audit/index.js') describe('routes > model > getModelCurrentUserPermissions', () => { diff --git a/backend/test/routes/model/getModelsSearch.spec.ts b/backend/test/routes/model/getModelsSearch.spec.ts index db52bcaa0..b770903a8 100644 --- a/backend/test/routes/model/getModelsSearch.spec.ts +++ b/backend/test/routes/model/getModelsSearch.spec.ts @@ -5,9 +5,7 @@ import audit from '../../../src/connectors/audit/__mocks__/index.js' import { getModelsSearchSchema } from '../../../src/routes/v2/model/getModelsSearch.js' import { createFixture, testGet } from '../../testUtils/routes.js' -vi.mock('../../../src/utils/config.js') vi.mock('../../../src/utils/user.js') -vi.mock('../../../src/utils/config.js') vi.mock('../../../src/connectors/audit/index.js') vi.mock('../../../src/services/model.js', () => ({ diff --git a/backend/test/routes/model/images/getImages.spec.ts b/backend/test/routes/model/images/getImages.spec.ts index d94362378..5124b944a 100644 --- a/backend/test/routes/model/images/getImages.spec.ts +++ b/backend/test/routes/model/images/getImages.spec.ts @@ -4,9 +4,7 @@ import audit from '../../../../src/connectors/audit/__mocks__/index.js' import { getImagesSchema } from '../../../../src/routes/v2/model/images/getImages.js' import { createFixture, testGet } from '../../../testUtils/routes.js' -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/utils/user.js') -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/connectors/audit/index.js') vi.mock('../../../../src/connectors/authorisation/index.js') diff --git a/backend/test/routes/model/inferencing/createInferencingService.spec.ts b/backend/test/routes/model/inferencing/createInferencingService.spec.ts index b5bd2b7ad..3f93fe093 100644 --- a/backend/test/routes/model/inferencing/createInferencingService.spec.ts +++ b/backend/test/routes/model/inferencing/createInferencingService.spec.ts @@ -4,7 +4,6 @@ import audit from '../../../../src/connectors/audit/__mocks__/index.js' import { postInferenceSchema } from '../../../../src/routes/v2/model/inferencing/postInferenceService.js' import { createFixture, testPost } from '../../../testUtils/routes.js' -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/utils/user.js') vi.mock('../../../../src/connectors/audit/index.js') vi.mock('../../../../src/connectors/authorisation/index.js') @@ -17,7 +16,7 @@ describe('routes > inferencing > postInference', () => { test('200 > ok', async () => { const fixture = createFixture(postInferenceSchema) const res = await testPost(`/api/v2/model/${fixture.params.modelId}/inference`, fixture) - + //console.log(res) expect(res.statusCode).toBe(200) expect(res.body).matchSnapshot() }) diff --git a/backend/test/routes/model/inferencing/getInferencingService.spec.ts b/backend/test/routes/model/inferencing/getInferencingService.spec.ts index d0e265754..ef89eb8fd 100644 --- a/backend/test/routes/model/inferencing/getInferencingService.spec.ts +++ b/backend/test/routes/model/inferencing/getInferencingService.spec.ts @@ -4,7 +4,6 @@ import audit from '../../../../src/connectors/audit/__mocks__/index.js' import { getInferenceSchema } from '../../../../src/routes/v2/model/inferencing/getInferenceService.js' import { createFixture, testGet } from '../../../testUtils/routes.js' -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/utils/user.js') vi.mock('../../../../src/connectors/audit/index.js') vi.mock('../../../../src/connectors/authorisation/index.js') diff --git a/backend/test/routes/model/inferencing/getInferencingServices.spec.ts b/backend/test/routes/model/inferencing/getInferencingServices.spec.ts index d77eeb2a7..85e9d5c29 100644 --- a/backend/test/routes/model/inferencing/getInferencingServices.spec.ts +++ b/backend/test/routes/model/inferencing/getInferencingServices.spec.ts @@ -4,7 +4,6 @@ import audit from '../../../../src/connectors/audit/__mocks__/index.js' import { getInferencesSchema } from '../../../../src/routes/v2/model/inferencing/getInferenceServices.js' import { createFixture, testGet } from '../../../testUtils/routes.js' -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/utils/user.js') vi.mock('../../../../src/connectors/audit/index.js') vi.mock('../../../../src/connectors/authorisation/index.js') diff --git a/backend/test/routes/model/inferencing/updateInferencingService.spec.ts b/backend/test/routes/model/inferencing/updateInferencingService.spec.ts index 962512ba7..fcd8e2a87 100644 --- a/backend/test/routes/model/inferencing/updateInferencingService.spec.ts +++ b/backend/test/routes/model/inferencing/updateInferencingService.spec.ts @@ -4,7 +4,6 @@ import audit from '../../../../src/connectors/audit/__mocks__/index.js' import { putInferenceSchema } from '../../../../src/routes/v2/model/inferencing/putInferenceService.js' import { createFixture, testPut } from '../../../testUtils/routes.js' -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/utils/user.js') vi.mock('../../../../src/connectors/audit/index.js') vi.mock('../../../../src/connectors/authorisation/index.js') diff --git a/backend/test/routes/model/modelcard/getModelCard.spec.ts b/backend/test/routes/model/modelcard/getModelCard.spec.ts index 69c619056..99121db5d 100644 --- a/backend/test/routes/model/modelcard/getModelCard.spec.ts +++ b/backend/test/routes/model/modelcard/getModelCard.spec.ts @@ -4,9 +4,7 @@ import audit from '../../../../src/connectors/audit/__mocks__/index.js' import { getModelCardSchema } from '../../../../src/routes/v2/model/modelcard/getModelCard.js' import { createFixture, testGet } from '../../../testUtils/routes.js' -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/utils/user.js') -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/connectors/audit/index.js') const modelMock = vi.hoisted(() => { diff --git a/backend/test/routes/model/modelcard/getModelCardHtml.spec.ts b/backend/test/routes/model/modelcard/getModelCardHtml.spec.ts index 9988635a0..628d1c07d 100644 --- a/backend/test/routes/model/modelcard/getModelCardHtml.spec.ts +++ b/backend/test/routes/model/modelcard/getModelCardHtml.spec.ts @@ -6,9 +6,7 @@ import { getModelCardHtmlSchema } from '../../../../src/routes/v2/model/modelcar import { renderToHtml } from '../../../../src/services/modelCardExport.js' import { createFixture, testGet } from '../../../testUtils/routes.js' -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/utils/user.js') -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/connectors/audit/index.js') const mockModelCardExportService = vi.hoisted(() => { diff --git a/backend/test/routes/model/modelcard/getModelCardRevisions.spec.ts b/backend/test/routes/model/modelcard/getModelCardRevisions.spec.ts index f1108487b..689ff1d2d 100644 --- a/backend/test/routes/model/modelcard/getModelCardRevisions.spec.ts +++ b/backend/test/routes/model/modelcard/getModelCardRevisions.spec.ts @@ -5,9 +5,7 @@ import { getModelCardRevisionsSchema } from '../../../../src/routes/v2/model/mod import { createFixture, testGet } from '../../../testUtils/routes.js' import { testModelCardRevision } from '../../../testUtils/testModels.js' -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/utils/user.js') -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/connectors/audit/index.js') const mockModelService = vi.hoisted(() => { diff --git a/backend/test/routes/model/modelcard/postFromSchema.spec.ts b/backend/test/routes/model/modelcard/postFromSchema.spec.ts index fb39c30b4..b1b4c531e 100644 --- a/backend/test/routes/model/modelcard/postFromSchema.spec.ts +++ b/backend/test/routes/model/modelcard/postFromSchema.spec.ts @@ -5,9 +5,7 @@ import { postFromSchemaSchema } from '../../../../src/routes/v2/model/modelcard/ import { createFixture, testPost } from '../../../testUtils/routes.js' vi.mock('../../../../src/connectors/authorisation/index.js') -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/utils/user.js') -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/connectors/audit/index.js') vi.mock('../../../../src/services/model.js', async () => { diff --git a/backend/test/routes/model/modelcard/postFromTemplate.spec.ts b/backend/test/routes/model/modelcard/postFromTemplate.spec.ts index b4e51d91a..3e98d83b3 100644 --- a/backend/test/routes/model/modelcard/postFromTemplate.spec.ts +++ b/backend/test/routes/model/modelcard/postFromTemplate.spec.ts @@ -5,9 +5,7 @@ import { postFromTemplateSchema } from '../../../../src/routes/v2/model/modelcar import { createFixture, testPost } from '../../../testUtils/routes.js' vi.mock('../../../../src/connectors/authorisation/index.js') -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/utils/user.js') -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/connectors/audit/index.js') vi.mock('../../../../src/services/model.js', async () => { diff --git a/backend/test/routes/model/modelcard/putModelCard.spec.ts b/backend/test/routes/model/modelcard/putModelCard.spec.ts index c5cc6c569..b8e2cfd84 100644 --- a/backend/test/routes/model/modelcard/putModelCard.spec.ts +++ b/backend/test/routes/model/modelcard/putModelCard.spec.ts @@ -5,9 +5,7 @@ import { putModelCardSchema } from '../../../../src/routes/v2/model/modelcard/pu import { createFixture, testPut } from '../../../testUtils/routes.js' vi.mock('../../../../src/connectors/authorisation/index.js') -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/utils/user.js') -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/connectors/audit/index.js') vi.mock('../../../../src/services/model.js', async () => { diff --git a/backend/test/routes/model/patchModel.spec.ts b/backend/test/routes/model/patchModel.spec.ts index 533e58638..75b5f8960 100644 --- a/backend/test/routes/model/patchModel.spec.ts +++ b/backend/test/routes/model/patchModel.spec.ts @@ -4,9 +4,7 @@ import audit from '../../../src/connectors/audit/__mocks__/index.js' import { patchModelSchema } from '../../../src/routes/v2/model/patchModel.js' import { createFixture, testPatch } from '../../testUtils/routes.js' -vi.mock('../../../src/utils/config.js') vi.mock('../../../src/utils/user.js') -vi.mock('../../../src/utils/config.js') vi.mock('../../../src/connectors/audit/index.js') describe('routes > model > patchModel', () => { diff --git a/backend/test/routes/model/postModel.spec.ts b/backend/test/routes/model/postModel.spec.ts index c0bb63836..71904c221 100644 --- a/backend/test/routes/model/postModel.spec.ts +++ b/backend/test/routes/model/postModel.spec.ts @@ -4,9 +4,7 @@ import audit from '../../../src/connectors/audit/__mocks__/index.js' import { postModelSchema } from '../../../src/routes/v2/model/postModel.js' import { createFixture, testPost } from '../../testUtils/routes.js' -vi.mock('../../../src/utils/config.js') vi.mock('../../../src/utils/user.js') -vi.mock('../../../src/utils/config.js') vi.mock('../../../src/connectors/audit/index.js') describe('routes > model > postModel', () => { diff --git a/backend/test/routes/model/postRequestExport.spec.ts b/backend/test/routes/model/postRequestExport.spec.ts index e25bb6d1e..5ff5a7acd 100644 --- a/backend/test/routes/model/postRequestExport.spec.ts +++ b/backend/test/routes/model/postRequestExport.spec.ts @@ -4,9 +4,7 @@ import audit from '../../../src/connectors/audit/__mocks__/index.js' import { postRequestExportToS3Schema } from '../../../src/routes/v2/model/postRequestExport.js' import { createFixture, testPost } from '../../testUtils/routes.js' -vi.mock('../../../src/utils/config.js') vi.mock('../../../src/utils/user.js') -vi.mock('../../../src/utils/config.js') vi.mock('../../../src/connectors/audit/index.js') describe('routes > model > postModel', () => { diff --git a/backend/test/routes/model/postRequestImport.spec.ts b/backend/test/routes/model/postRequestImport.spec.ts index 02be87632..83c5e71bb 100644 --- a/backend/test/routes/model/postRequestImport.spec.ts +++ b/backend/test/routes/model/postRequestImport.spec.ts @@ -4,9 +4,7 @@ import audit from '../../../src/connectors/audit/__mocks__/index.js' import { postRequestImportFromS3Schema } from '../../../src/routes/v2/model/postRequestImport.js' import { createFixture, testPost } from '../../testUtils/routes.js' -vi.mock('../../../src/utils/config.js') vi.mock('../../../src/utils/user.js') -vi.mock('../../../src/utils/config.js') vi.mock('../../../src/connectors/audit/index.js') describe('routes > model > postRequestImport', () => { diff --git a/backend/test/routes/model/release/deleteRelease.spec.ts b/backend/test/routes/model/release/deleteRelease.spec.ts index 85da8fc43..eb888c5b4 100644 --- a/backend/test/routes/model/release/deleteRelease.spec.ts +++ b/backend/test/routes/model/release/deleteRelease.spec.ts @@ -4,9 +4,7 @@ import audit from '../../../../src/connectors/audit/__mocks__/index.js' import { deleteReleaseSchema } from '../../../../src/routes/v2/release/deleteRelease.js' import { createFixture, testDelete } from '../../../testUtils/routes.js' -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/utils/user.js') -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/connectors/audit/index.js') vi.mock('../../../../src/connectors/authorisation/index.js') diff --git a/backend/test/routes/model/release/getRelease.spec.ts b/backend/test/routes/model/release/getRelease.spec.ts index 44951614c..93249d36e 100644 --- a/backend/test/routes/model/release/getRelease.spec.ts +++ b/backend/test/routes/model/release/getRelease.spec.ts @@ -4,9 +4,7 @@ import audit from '../../../../src/connectors/audit/__mocks__/index.js' import { getReleaseSchema } from '../../../../src/routes/v2/release/getRelease.js' import { createFixture, testGet } from '../../../testUtils/routes.js' -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/utils/user.js') -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/connectors/audit/index.js') vi.mock('../../../../src/connectors/authorisation/index.js') diff --git a/backend/test/routes/model/release/getReleases.spec.ts b/backend/test/routes/model/release/getReleases.spec.ts index 1a83ca99c..b859a8582 100644 --- a/backend/test/routes/model/release/getReleases.spec.ts +++ b/backend/test/routes/model/release/getReleases.spec.ts @@ -4,9 +4,7 @@ import audit from '../../../../src/connectors/audit/__mocks__/index.js' import { getReleasesSchema } from '../../../../src/routes/v2/release/getReleases.js' import { createFixture, testGet } from '../../../testUtils/routes.js' -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/utils/user.js') -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/connectors/audit/index.js') vi.mock('../../../../src/connectors/authorisation/index.js') diff --git a/backend/test/routes/model/release/postRelease.spec.ts b/backend/test/routes/model/release/postRelease.spec.ts index 28dae9343..10b8c76f5 100644 --- a/backend/test/routes/model/release/postRelease.spec.ts +++ b/backend/test/routes/model/release/postRelease.spec.ts @@ -4,9 +4,7 @@ import audit from '../../../../src/connectors/audit/__mocks__/index.js' import { postReleaseSchema } from '../../../../src/routes/v2/release/postRelease.js' import { createFixture, testPost } from '../../../testUtils/routes.js' -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/utils/user.js') -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/connectors/audit/index.js') vi.mock('../../../../src/connectors/authorisation/index.js') diff --git a/backend/test/routes/model/release/postReleaseComment.spec.ts b/backend/test/routes/model/release/postReleaseComment.spec.ts index 036b9a599..cd99745f9 100644 --- a/backend/test/routes/model/release/postReleaseComment.spec.ts +++ b/backend/test/routes/model/release/postReleaseComment.spec.ts @@ -4,8 +4,6 @@ import audit from '../../../../src/connectors/audit/__mocks__/index.js' import { postReleaseCommentSchema } from '../../../../src/routes/v2/release/postReleaseComment.js' import { createFixture, testPost } from '../../../testUtils/routes.js' -vi.mock('../../../../src/utils/config.js') -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/utils/user.js') vi.mock('../../../../src/connectors/audit/index.js') vi.mock('../../../../src/connectors/authorisation/index.js') diff --git a/backend/test/routes/model/release/putRelease.spec.ts b/backend/test/routes/model/release/putRelease.spec.ts index 613021270..951baef2c 100644 --- a/backend/test/routes/model/release/putRelease.spec.ts +++ b/backend/test/routes/model/release/putRelease.spec.ts @@ -4,9 +4,7 @@ import audit from '../../../../src/connectors/audit/__mocks__/index.js' import { putReleaseSchema } from '../../../../src/routes/v2/release/putRelease.js' import { createFixture, testPut } from '../../../testUtils/routes.js' -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/utils/user.js') -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/connectors/audit/index.js') vi.mock('../../../../src/connectors/authorisation/index.js') diff --git a/backend/test/routes/model/webhook/deleteWebhook.spec.ts b/backend/test/routes/model/webhook/deleteWebhook.spec.ts index 0d054c252..c2f703f4a 100644 --- a/backend/test/routes/model/webhook/deleteWebhook.spec.ts +++ b/backend/test/routes/model/webhook/deleteWebhook.spec.ts @@ -2,9 +2,7 @@ import { describe, expect, test, vi } from 'vitest' import { testDelete } from '../../../testUtils/routes.js' -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/utils/user.js') -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/connectors/audit/index.js') vi.mock('../../../../src/connectors/authorisation/index.js') diff --git a/backend/test/routes/model/webhook/getWebhooks.spec.ts b/backend/test/routes/model/webhook/getWebhooks.spec.ts index 9df044817..6a6e8d64f 100644 --- a/backend/test/routes/model/webhook/getWebhooks.spec.ts +++ b/backend/test/routes/model/webhook/getWebhooks.spec.ts @@ -2,9 +2,7 @@ import { describe, expect, test, vi } from 'vitest' import { testGet } from '../../../testUtils/routes.js' -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/utils/user.js') -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/connectors/audit/index.js') vi.mock('../../../../src/connectors/authorisation/index.js') diff --git a/backend/test/routes/model/webhook/postWebhook.spec.ts b/backend/test/routes/model/webhook/postWebhook.spec.ts index e1ef2abd0..131c5b67e 100644 --- a/backend/test/routes/model/webhook/postWebhook.spec.ts +++ b/backend/test/routes/model/webhook/postWebhook.spec.ts @@ -3,9 +3,7 @@ import { describe, expect, test, vi } from 'vitest' import { postWebhookSchema } from '../../../../src/routes/v2/model/webhook/postWebhook.js' import { createFixture, testPost } from '../../../testUtils/routes.js' -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/utils/user.js') -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/connectors/audit/index.js') vi.mock('../../../../src/connectors/authorisation/index.js') diff --git a/backend/test/routes/model/webhook/putWebhook.spec.ts b/backend/test/routes/model/webhook/putWebhook.spec.ts index 2b28bc185..829916740 100644 --- a/backend/test/routes/model/webhook/putWebhook.spec.ts +++ b/backend/test/routes/model/webhook/putWebhook.spec.ts @@ -3,9 +3,7 @@ import { describe, expect, test, vi } from 'vitest' import { putWebhookSchema } from '../../../../src/routes/v2/model/webhook/putWebhook.js' import { createFixture, testPut } from '../../../testUtils/routes.js' -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/utils/user.js') -vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/connectors/audit/index.js') vi.mock('../../../../src/connectors/authorisation/index.js') diff --git a/backend/test/routes/review/getReviews.spec.ts b/backend/test/routes/review/getReviews.spec.ts index 3b7b08ac5..45d057d7d 100644 --- a/backend/test/routes/review/getReviews.spec.ts +++ b/backend/test/routes/review/getReviews.spec.ts @@ -4,8 +4,6 @@ import audit from '../../../src/connectors/audit/__mocks__/index.js' import { testGet } from '../../testUtils/routes.js' import { testReleaseReview } from '../../testUtils/testModels.js' -vi.mock('../../../src/utils/config.js') -vi.mock('../../../src/utils/config.js') vi.mock('../../../src/utils/user.js') vi.mock('../../../src/connectors/audit/index.js') vi.mock('../../../src/connectors/authorisation/index.js') diff --git a/backend/test/routes/review/postAccessRequestReviewResponse.spec.ts b/backend/test/routes/review/postAccessRequestReviewResponse.spec.ts index 303da9b63..0b0fe7a4a 100644 --- a/backend/test/routes/review/postAccessRequestReviewResponse.spec.ts +++ b/backend/test/routes/review/postAccessRequestReviewResponse.spec.ts @@ -6,8 +6,6 @@ import { postAccessRequestReviewResponseSchema } from '../../../src/routes/v2/re import { createFixture, testPost } from '../../testUtils/routes.js' import { testReviewResponse } from '../../testUtils/testModels.js' -vi.mock('../../../src/utils/config.js') -vi.mock('../../../src/utils/config.js') vi.mock('../../../src/utils/user.js') vi.mock('../../../src/connectors/audit/index.js') vi.mock('../../../src/connectors/authorisation/index.js') diff --git a/backend/test/routes/review/postReleaseReviewResponse.spec.ts b/backend/test/routes/review/postReleaseReviewResponse.spec.ts index 894863162..cad90d558 100644 --- a/backend/test/routes/review/postReleaseReviewResponse.spec.ts +++ b/backend/test/routes/review/postReleaseReviewResponse.spec.ts @@ -6,8 +6,6 @@ import { postReleaseReviewResponseSchema } from '../../../src/routes/v2/review/p import { createFixture, testPost } from '../../testUtils/routes.js' import { testReviewResponse } from '../../testUtils/testModels.js' -vi.mock('../../../src/utils/config.js') -vi.mock('../../../src/utils/config.js') vi.mock('../../../src/utils/user.js') vi.mock('../../../src/connectors/audit/index.js') vi.mock('../../../src/connectors/authorisation/index.js') diff --git a/backend/test/routes/schema/deleteSchema.spec.ts b/backend/test/routes/schema/deleteSchema.spec.ts index 2866c3dcc..2bea8deec 100644 --- a/backend/test/routes/schema/deleteSchema.spec.ts +++ b/backend/test/routes/schema/deleteSchema.spec.ts @@ -5,8 +5,6 @@ import { deleteSchemaSchema } from '../../../src/routes/v2/schema/deleteSchema.j import { createFixture, testDelete } from '../../testUtils/routes.js' vi.mock('../../../src/utils/user.js') -vi.mock('../../../src/utils/config.js') -vi.mock('../../../src/utils/config.js') vi.mock('../../../src/connectors/audit/index.js') vi.mock('../../../src/connectors/authorisation/index.js') diff --git a/backend/test/routes/schema/getSchema.spec.ts b/backend/test/routes/schema/getSchema.spec.ts index 89e473d0e..41c140822 100644 --- a/backend/test/routes/schema/getSchema.spec.ts +++ b/backend/test/routes/schema/getSchema.spec.ts @@ -5,9 +5,7 @@ import { NotFound } from '../../../src/utils/error.js' import { testGet } from '../../testUtils/routes.js' import { testModelSchema } from '../../testUtils/testModels.js' -vi.mock('../../../src/utils/config.js') vi.mock('../../../src/utils/user.js') -vi.mock('../../../src/utils/config.js') vi.mock('../../../src/connectors/audit/index.js') vi.mock('../../../src/connectors/authorisation/index.js') diff --git a/backend/test/routes/schema/getSchemas.spec.ts b/backend/test/routes/schema/getSchemas.spec.ts index 0dca0e02c..930e00b1b 100644 --- a/backend/test/routes/schema/getSchemas.spec.ts +++ b/backend/test/routes/schema/getSchemas.spec.ts @@ -4,9 +4,7 @@ import audit from '../../../src/connectors/audit/__mocks__/index.js' import { testGet } from '../../testUtils/routes.js' import { testDeploymentSchema, testModelSchema } from '../../testUtils/testModels.js' -vi.mock('../../../src/utils/config.js') vi.mock('../../../src/utils/user.js') -vi.mock('../../../src/utils/config.js') vi.mock('../../../src/connectors/audit/index.js') vi.mock('../../../src/connectors/authorisation/index.js') diff --git a/backend/test/routes/schema/patchSchema.spec.ts b/backend/test/routes/schema/patchSchema.spec.ts index f3c0813df..299dd3f4e 100644 --- a/backend/test/routes/schema/patchSchema.spec.ts +++ b/backend/test/routes/schema/patchSchema.spec.ts @@ -5,8 +5,6 @@ import { patchSchemaSchema } from '../../../src/routes/v2/schema/patchSchema.js' import { createFixture, testPatch } from '../../testUtils/routes.js' vi.mock('../../../src/utils/user.js') -vi.mock('../../../src/utils/config.js') -vi.mock('../../../src/utils/config.js') vi.mock('../../../src/connectors/audit/index.js') vi.mock('../../../src/connectors/authorisation/index.js') diff --git a/backend/test/routes/schema/postSchema.spec.ts b/backend/test/routes/schema/postSchema.spec.ts index 73118078b..928af27de 100644 --- a/backend/test/routes/schema/postSchema.spec.ts +++ b/backend/test/routes/schema/postSchema.spec.ts @@ -5,8 +5,6 @@ import { postSchemaSchema } from '../../../src/routes/v2/schema/postSchema.js' import { createFixture, testPost } from '../../testUtils/routes.js' vi.mock('../../../src/utils/user.js') -vi.mock('../../../src/utils/config.js') -vi.mock('../../../src/utils/config.js') vi.mock('../../../src/connectors/audit/index.js') vi.mock('../../../src/connectors/authorisation/index.js') diff --git a/backend/test/services/file.spec.ts b/backend/test/services/file.spec.ts index 90a16d82c..99df49326 100644 --- a/backend/test/services/file.spec.ts +++ b/backend/test/services/file.spec.ts @@ -57,13 +57,12 @@ const fileScanResult: FileScanResult = { isInfected: false, toolName: 'Test', } + const fileScanningMock = vi.hoisted(() => ({ - default: vi.fn(() => ({ - info: vi.fn(() => []), - scan: vi.fn(() => new Promise(() => fileScanResult)), - })), + info: vi.fn(() => []), + scan: vi.fn(() => new Promise(() => [fileScanResult])), })) -vi.mock('../../src/connectors/fileScanning/index.js', async () => fileScanningMock) +vi.mock('../../src/connectors/fileScanning/index.js', async () => ({ default: fileScanningMock })) const s3Mocks = vi.hoisted(() => ({ putObjectStream: vi.fn(() => ({ fileSize: 100 })), @@ -114,7 +113,6 @@ describe('services > file', () => { const mime = 'text/plain' const stream = new Readable() as any fileModelMocks.save.mockResolvedValueOnce({ example: true }) - const result = await uploadFile(user, modelId, name, mime, stream) expect(s3Mocks.putObjectStream).toBeCalled() diff --git a/backend/test/services/mirroredModel.spec.ts b/backend/test/services/mirroredModel.spec.ts index 27defa6d1..74fc2b325 100644 --- a/backend/test/services/mirroredModel.spec.ts +++ b/backend/test/services/mirroredModel.spec.ts @@ -2,15 +2,21 @@ import { PassThrough } from 'stream' import { describe, expect, test, vi } from 'vitest' import authorisation from '../../src/connectors/authorisation/index.js' +import { FileScanResult } from '../../src/connectors/fileScanning/Base.js' import { UserInterface } from '../../src/models/User.js' import { exportModel, importModel } from '../../src/services/mirroredModel.js' +const fileScanResult: FileScanResult = { + state: 'complete', + isInfected: false, + toolName: 'Test', +} + const fileScanningMock = vi.hoisted(() => ({ - default: vi.fn(() => ({ - info: vi.fn(() => []), - })), + info: vi.fn(() => []), + scan: vi.fn(() => new Promise(() => [fileScanResult])), })) -vi.mock('../../src/connectors/fileScanning/index.js', async () => fileScanningMock) +vi.mock('../../src/connectors/fileScanning/index.js', async () => ({ default: fileScanningMock })) const fflateMock = vi.hoisted(() => ({ unzipSync: vi.fn(), @@ -34,9 +40,6 @@ vi.mock('../../src/connectors/authorisation/index.js', async () => ({ default: authMock, })) -// const fileScanConnectors: BaseFileScanningConnector[] = [new ClamAvFileScanningConnector()] -// const wrapper = new FileScanningWrapper(fileScanConnectors) - const configMock = vi.hoisted( () => ({ From 7e6b5f629f0ae77411f8e0080f420d02af5dbd5f Mon Sep 17 00:00:00 2001 From: araddcc002 Date: Wed, 30 Oct 2024 12:00:37 +0000 Subject: [PATCH 29/32] updated bailo config map for filescanners --- infrastructure/helm/bailo/templates/bailo/bailo.configmap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/helm/bailo/templates/bailo/bailo.configmap.yaml b/infrastructure/helm/bailo/templates/bailo/bailo.configmap.yaml index 3e5302367..17570373e 100644 --- a/infrastructure/helm/bailo/templates/bailo/bailo.configmap.yaml +++ b/infrastructure/helm/bailo/templates/bailo/bailo.configmap.yaml @@ -194,7 +194,7 @@ data: }, fileScanners: { - kinds: '{{ .Values.connectors.fileScanners.kinds }}', + kinds: {{ toJson .Values.connectors.fileScanners.kinds }} }, }, From 62a66c28dfa0e274e6f3078804d320198531cbe8 Mon Sep 17 00:00:00 2001 From: araddcc002 Date: Wed, 30 Oct 2024 13:36:30 +0000 Subject: [PATCH 30/32] improved checking for scanners info before show av chip on file displays --- backend/config/docker_compose.cjs | 2 +- frontend/actions/fileScanning.ts | 2 +- frontend/src/entry/model/releases/FileDownload.tsx | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/config/docker_compose.cjs b/backend/config/docker_compose.cjs index d113374d1..497ffdfb6 100644 --- a/backend/config/docker_compose.cjs +++ b/backend/config/docker_compose.cjs @@ -69,7 +69,7 @@ module.exports = { connectors: { fileScanners: { - kinds: ['clamAV'], + kinds: [], }, }, } diff --git a/frontend/actions/fileScanning.ts b/frontend/actions/fileScanning.ts index 8ec44e068..c3732c9e3 100644 --- a/frontend/actions/fileScanning.ts +++ b/frontend/actions/fileScanning.ts @@ -13,7 +13,7 @@ export function useGetFileScannerInfo() { return { scannersMutate: mutate, - scanners: data ? data : emptyScannerList, + scanners: data ? data.scanners : emptyScannerList, isScannersLoading: isLoading, isScannersError: error, } diff --git a/frontend/src/entry/model/releases/FileDownload.tsx b/frontend/src/entry/model/releases/FileDownload.tsx index 578b13266..583e4e715 100644 --- a/frontend/src/entry/model/releases/FileDownload.tsx +++ b/frontend/src/entry/model/releases/FileDownload.tsx @@ -136,7 +136,7 @@ export default function FileDownload({ modelId, file }: FileDownloadProps) { - {scanners && avChip} + {scanners.length > 0 && avChip} From 610791bab77e18b9997b8a232a2dc5ef4e9291e4 Mon Sep 17 00:00:00 2001 From: araddcc002 Date: Wed, 30 Oct 2024 13:45:51 +0000 Subject: [PATCH 31/32] readded clamav to config --- backend/config/docker_compose.cjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/config/docker_compose.cjs b/backend/config/docker_compose.cjs index 497ffdfb6..d113374d1 100644 --- a/backend/config/docker_compose.cjs +++ b/backend/config/docker_compose.cjs @@ -69,7 +69,7 @@ module.exports = { connectors: { fileScanners: { - kinds: [], + kinds: ['clamAV'], }, }, } From c3362fadc2d83d5d3737bbefae4c592f9d7811ff Mon Sep 17 00:00:00 2001 From: araddcc002 Date: Wed, 30 Oct 2024 14:02:32 +0000 Subject: [PATCH 32/32] updated test mocking --- backend/config/test.cjs | 13 +------------ .../inferencing/createInferencingService.spec.ts | 2 +- .../model/inferencing/getInferencingService.spec.ts | 1 + .../inferencing/getInferencingServices.spec.ts | 1 + .../inferencing/updateInferencingService.spec.ts | 1 + 5 files changed, 5 insertions(+), 13 deletions(-) diff --git a/backend/config/test.cjs b/backend/config/test.cjs index 688cfa41b..4ba52ba2c 100644 --- a/backend/config/test.cjs +++ b/backend/config/test.cjs @@ -1,12 +1 @@ -module.exports = { - ui: { - inference: { - enabled: true, - }, - }, - connectors: { - fileScanners: { - kinds: [], - }, - }, -} +module.exports = {} diff --git a/backend/test/routes/model/inferencing/createInferencingService.spec.ts b/backend/test/routes/model/inferencing/createInferencingService.spec.ts index 3f93fe093..6b69bc814 100644 --- a/backend/test/routes/model/inferencing/createInferencingService.spec.ts +++ b/backend/test/routes/model/inferencing/createInferencingService.spec.ts @@ -4,6 +4,7 @@ import audit from '../../../../src/connectors/audit/__mocks__/index.js' import { postInferenceSchema } from '../../../../src/routes/v2/model/inferencing/postInferenceService.js' import { createFixture, testPost } from '../../../testUtils/routes.js' +vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/utils/user.js') vi.mock('../../../../src/connectors/audit/index.js') vi.mock('../../../../src/connectors/authorisation/index.js') @@ -16,7 +17,6 @@ describe('routes > inferencing > postInference', () => { test('200 > ok', async () => { const fixture = createFixture(postInferenceSchema) const res = await testPost(`/api/v2/model/${fixture.params.modelId}/inference`, fixture) - //console.log(res) expect(res.statusCode).toBe(200) expect(res.body).matchSnapshot() }) diff --git a/backend/test/routes/model/inferencing/getInferencingService.spec.ts b/backend/test/routes/model/inferencing/getInferencingService.spec.ts index ef89eb8fd..dcc68af18 100644 --- a/backend/test/routes/model/inferencing/getInferencingService.spec.ts +++ b/backend/test/routes/model/inferencing/getInferencingService.spec.ts @@ -5,6 +5,7 @@ import { getInferenceSchema } from '../../../../src/routes/v2/model/inferencing/ import { createFixture, testGet } from '../../../testUtils/routes.js' vi.mock('../../../../src/utils/user.js') +vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/connectors/audit/index.js') vi.mock('../../../../src/connectors/authorisation/index.js') diff --git a/backend/test/routes/model/inferencing/getInferencingServices.spec.ts b/backend/test/routes/model/inferencing/getInferencingServices.spec.ts index 85e9d5c29..d77eeb2a7 100644 --- a/backend/test/routes/model/inferencing/getInferencingServices.spec.ts +++ b/backend/test/routes/model/inferencing/getInferencingServices.spec.ts @@ -4,6 +4,7 @@ import audit from '../../../../src/connectors/audit/__mocks__/index.js' import { getInferencesSchema } from '../../../../src/routes/v2/model/inferencing/getInferenceServices.js' import { createFixture, testGet } from '../../../testUtils/routes.js' +vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/utils/user.js') vi.mock('../../../../src/connectors/audit/index.js') vi.mock('../../../../src/connectors/authorisation/index.js') diff --git a/backend/test/routes/model/inferencing/updateInferencingService.spec.ts b/backend/test/routes/model/inferencing/updateInferencingService.spec.ts index fcd8e2a87..962512ba7 100644 --- a/backend/test/routes/model/inferencing/updateInferencingService.spec.ts +++ b/backend/test/routes/model/inferencing/updateInferencingService.spec.ts @@ -4,6 +4,7 @@ import audit from '../../../../src/connectors/audit/__mocks__/index.js' import { putInferenceSchema } from '../../../../src/routes/v2/model/inferencing/putInferenceService.js' import { createFixture, testPut } from '../../../testUtils/routes.js' +vi.mock('../../../../src/utils/config.js') vi.mock('../../../../src/utils/user.js') vi.mock('../../../../src/connectors/audit/index.js') vi.mock('../../../../src/connectors/authorisation/index.js')