From 68e11ecbbcc42a4edb85317fc2d78ecc628b6c08 Mon Sep 17 00:00:00 2001 From: JR40159 <126243293+JR40159@users.noreply.github.com> Date: Tue, 9 Jan 2024 16:47:20 +0000 Subject: [PATCH 1/7] Add initial migration of versions --- backend/config/default.cjs | 2 +- backend/src/scripts/migrateV2.ts | 79 ++++++++++++++++++++++++++++++-- 2 files changed, 77 insertions(+), 4 deletions(-) diff --git a/backend/config/default.cjs b/backend/config/default.cjs index 8dde3fd71..488e64f60 100644 --- a/backend/config/default.cjs +++ b/backend/config/default.cjs @@ -40,7 +40,7 @@ module.exports = { mongo: { // A mongo connection URI, can contain usernames, passwords, replica set information, etc. // See: https://www.mongodb.com/docs/manual/reference/connection-string/ - uri: 'mongodb://localhost:27017/bailo', + uri: 'mongodb://localhost:27017/bailo?directConnection=true', }, minio: { diff --git a/backend/src/scripts/migrateV2.ts b/backend/src/scripts/migrateV2.ts index bd329e627..846dcf9d2 100644 --- a/backend/src/scripts/migrateV2.ts +++ b/backend/src/scripts/migrateV2.ts @@ -2,10 +2,13 @@ import { uniqWith } from 'lodash-es' import DeploymentModelV1 from '../models/Deployment.js' import ModelModelV1 from '../models/Model.js' +import FileModel from '../models/v2/File.js' import ModelModelV2, { CollaboratorEntry, ModelVisibility } from '../models/v2/Model.js' import ModelCardRevisionV2 from '../models/v2/ModelCardRevision.js' +import Release from '../models/v2/Release.js' import VersionModelV1 from '../models/Version.js' import { VersionDoc } from '../types/types.js' +import config from '../utils/config.js' import { connectToMongoose, disconnectFromMongoose } from '../utils/database.js' const MODEL_SCHEMA_MAP = { @@ -49,7 +52,7 @@ export async function migrateAllModels() { } async function migrateModel(modelId: string) { - const model = await ModelModelV1.findOne({ uuid: modelId }).populate('latestVersion') + const model = await ModelModelV1.findOne({ _id: modelId }).populate('latestVersion') if (!model) throw new Error(`Model not found: ${modelId}`) model.latestVersion = model.latestVersion as VersionDoc @@ -153,6 +156,76 @@ async function migrateModel(modelId: string) { timestamps: false, }, ) + + const v2Files: string[] = [] + const bucket = config.minio.buckets.uploads + if (version.files.rawBinaryPath) { + const file = await new FileModel({ + modelId, + name: `${version.version}-rawBinaryPath.zip`, + size: 0, //TODO + mime: 'application/x-zip-compressed', + bucket, + path: version.files.rawBinaryPath, + complete: true, + }) + await file.save() + v2Files.push(file._id.toString()) + } + if (version.files.rawCodePath) { + const file = await new FileModel({ + modelId, + name: `${version.version}-rawCodePath.zip`, + size: 1911, //TODO + mime: 'application/x-zip-compressed', + bucket, + path: version.files.rawCodePath, + complete: true, + }) + await file.save() + v2Files.push(file._id.toString()) + } + if (version.files.rawDockerPath) { + const file = await new FileModel({ + modelId, + name: `${version.version}-rawDockerPath`, + size: 0, //TODO + mime: 'application/octet-stream', + bucket, + path: version.files.rawDockerPath, + complete: true, + }) + await file.save() + v2Files.push(file._id.toString()) + } + + await Release.findOneAndUpdate( + { modelId, semver: `0.0.${i + 1}` }, + { + modelId, + modelCardVersion: i, + + semver: `0.0.${i + 1}`, + notes: `Migrated from V1. Orginal version ID: ${version.version}`, + + minor: false, + draft: false, + + fileIds: v2Files, + // Not sure about images + images: [], + + createdBy: 'system', + createdAt: version.createdAt, + updatedAt: version.updatedAt, + }, + + { + new: true, + upsert: true, // Make this update into an upsert + timestamps: false, + }, + ) } modelV2.card = { @@ -171,7 +244,7 @@ async function migrateModel(modelId: string) { await connectToMongoose() -// await migrateAllModels() -await migrateModel('minimal-model-for-testing-im7q59') +await migrateAllModels() +//await migrateModel('minimal-model-for-testing-im7q59') setTimeout(disconnectFromMongoose, 500) From 258a8b9096540b69b1225a99e9f10d625dbeacd3 Mon Sep 17 00:00:00 2001 From: JR40159 <126243293+JR40159@users.noreply.github.com> Date: Fri, 12 Jan 2024 10:36:18 +0000 Subject: [PATCH 2/7] Get file sizes --- backend/src/clients/s3.ts | 16 +++++++++++++++- backend/src/scripts/migrateV2.ts | 28 +++++++++++++++++++--------- backend/test/clients/s3.spec.ts | 16 +++++++++++++++- 3 files changed, 49 insertions(+), 11 deletions(-) diff --git a/backend/src/clients/s3.ts b/backend/src/clients/s3.ts index 13f7e1247..5ae3835d7 100644 --- a/backend/src/clients/s3.ts +++ b/backend/src/clients/s3.ts @@ -1,4 +1,4 @@ -import { GetObjectCommand, GetObjectRequest, S3Client } from '@aws-sdk/client-s3' +import { GetObjectCommand, GetObjectRequest, HeadObjectRequest, S3Client } from '@aws-sdk/client-s3' import { Upload } from '@aws-sdk/lib-storage' import { NodeHttpHandler } from '@smithy/node-http-handler' @@ -57,3 +57,17 @@ export async function getObjectStream(bucket: string, key: string, range?: { sta return response } + +export async function headObject(bucket: string, key: string) { + const client = await getS3Client() + + const input: HeadObjectRequest = { + Bucket: bucket, + Key: key, + } + + const command = new GetObjectCommand(input) + const response = await client.send(command) + + return response +} diff --git a/backend/src/scripts/migrateV2.ts b/backend/src/scripts/migrateV2.ts index 846dcf9d2..c461cfd47 100644 --- a/backend/src/scripts/migrateV2.ts +++ b/backend/src/scripts/migrateV2.ts @@ -1,5 +1,6 @@ import { uniqWith } from 'lodash-es' +import { headObject } from '../clients/s3.js' import DeploymentModelV1 from '../models/Deployment.js' import ModelModelV1 from '../models/Model.js' import FileModel from '../models/v2/File.js' @@ -163,40 +164,49 @@ async function migrateModel(modelId: string) { const file = await new FileModel({ modelId, name: `${version.version}-rawBinaryPath.zip`, - size: 0, //TODO mime: 'application/x-zip-compressed', bucket, path: version.files.rawBinaryPath, complete: true, }) - await file.save() - v2Files.push(file._id.toString()) + const size = await (await headObject(bucket, file.path)).ContentLength + if (size) { + file.size = size + await file.save() + v2Files.push(file._id.toString()) + } } if (version.files.rawCodePath) { const file = await new FileModel({ modelId, name: `${version.version}-rawCodePath.zip`, - size: 1911, //TODO mime: 'application/x-zip-compressed', bucket, path: version.files.rawCodePath, complete: true, }) - await file.save() - v2Files.push(file._id.toString()) + const size = await (await headObject(bucket, file.path)).ContentLength + if (size) { + file.size = size + await file.save() + v2Files.push(file._id.toString()) + } } if (version.files.rawDockerPath) { const file = await new FileModel({ modelId, name: `${version.version}-rawDockerPath`, - size: 0, //TODO mime: 'application/octet-stream', bucket, path: version.files.rawDockerPath, complete: true, }) - await file.save() - v2Files.push(file._id.toString()) + const size = await (await headObject(bucket, file.path)).ContentLength + if (size) { + file.size = size + await file.save() + v2Files.push(file._id.toString()) + } } await Release.findOneAndUpdate( diff --git a/backend/test/clients/s3.spec.ts b/backend/test/clients/s3.spec.ts index be176fb2a..bbd606f5a 100644 --- a/backend/test/clients/s3.spec.ts +++ b/backend/test/clients/s3.spec.ts @@ -1,6 +1,6 @@ import { describe, expect, test, vi } from 'vitest' -import { getObjectStream } from '../../src/clients/s3.js' +import { getObjectStream, headObject } from '../../src/clients/s3.js' const s3Mocks = vi.hoisted(() => { const send = vi.fn(() => 'response') @@ -29,4 +29,18 @@ describe('clients > s3', () => { expect(s3Mocks.send).toHaveBeenCalled() expect(response).toBe('response') }) + + test('headObject > success', async () => { + const bucket = 'test-bucket' + const key = 'test-key' + + const response = await headObject(bucket, key) + + expect(s3Mocks.GetObjectCommand).toHaveBeenCalledWith({ + Bucket: bucket, + Key: key, + }) + expect(s3Mocks.send).toHaveBeenCalled() + expect(response).toBe('response') + }) }) From 345af966d237f15bdbdf0b2d9614f44f18675801 Mon Sep 17 00:00:00 2001 From: JR40159 <126243293+JR40159@users.noreply.github.com> Date: Fri, 12 Jan 2024 12:11:26 +0000 Subject: [PATCH 3/7] Don't add file if it can't be found in s3 --- backend/src/clients/s3.ts | 6 +- backend/src/scripts/migrateV2.ts | 120 +++++++++++++++++++++---------- 2 files changed, 88 insertions(+), 38 deletions(-) diff --git a/backend/src/clients/s3.ts b/backend/src/clients/s3.ts index 5ae3835d7..b3f42a55f 100644 --- a/backend/src/clients/s3.ts +++ b/backend/src/clients/s3.ts @@ -1,4 +1,4 @@ -import { GetObjectCommand, GetObjectRequest, HeadObjectRequest, S3Client } from '@aws-sdk/client-s3' +import { GetObjectCommand, GetObjectRequest, HeadObjectRequest, NoSuchKey, S3Client } from '@aws-sdk/client-s3' import { Upload } from '@aws-sdk/lib-storage' import { NodeHttpHandler } from '@smithy/node-http-handler' @@ -71,3 +71,7 @@ export async function headObject(bucket: string, key: string) { return response } + +export function isNoSuchKeyException(err: any): err is NoSuchKey { + return err?.name === 'NoSuchKey' +} diff --git a/backend/src/scripts/migrateV2.ts b/backend/src/scripts/migrateV2.ts index c461cfd47..acf0cf7c9 100644 --- a/backend/src/scripts/migrateV2.ts +++ b/backend/src/scripts/migrateV2.ts @@ -1,6 +1,6 @@ import { uniqWith } from 'lodash-es' -import { headObject } from '../clients/s3.js' +import { headObject, isNoSuchKeyException } from '../clients/s3.js' import DeploymentModelV1 from '../models/Deployment.js' import ModelModelV1 from '../models/Model.js' import FileModel from '../models/v2/File.js' @@ -41,6 +41,19 @@ const _DEPLOYMENT_SCHEMA_MAP = { }, } +async function getObjectContentLength(bucket: string, object: string) { + let objectMeta + try { + objectMeta = await headObject(bucket, object) + } catch (e) { + if (isNoSuchKeyException(e)) { + return + } + throw e + } + return objectMeta.ContentLength +} + function identityConversion(old: string) { return old } @@ -161,51 +174,84 @@ async function migrateModel(modelId: string) { const v2Files: string[] = [] const bucket = config.minio.buckets.uploads if (version.files.rawBinaryPath) { - const file = await new FileModel({ - modelId, - name: `${version.version}-rawBinaryPath.zip`, - mime: 'application/x-zip-compressed', - bucket, - path: version.files.rawBinaryPath, - complete: true, - }) - const size = await (await headObject(bucket, file.path)).ContentLength + const path = version.files.rawBinaryPath + const size = await getObjectContentLength(bucket, path) if (size) { - file.size = size - await file.save() - v2Files.push(file._id.toString()) + const v2File = await FileModel.findOneAndUpdate( + { modelId, bucket, path }, + { + modelId, + name: `${version.version}-rawBinaryPath.zip`, + mime: 'application/x-zip-compressed', + size, + bucket, + path, + complete: true, + + createdAt: version.createdAt, + updatedAt: version.updatedAt, + }, + { + new: true, + upsert: true, // Make this update into an upsert + timestamps: false, + }, + ) + v2Files.push(v2File._id.toString()) } } if (version.files.rawCodePath) { - const file = await new FileModel({ - modelId, - name: `${version.version}-rawCodePath.zip`, - mime: 'application/x-zip-compressed', - bucket, - path: version.files.rawCodePath, - complete: true, - }) - const size = await (await headObject(bucket, file.path)).ContentLength + const path = version.files.rawCodePath + const size = await getObjectContentLength(bucket, path) if (size) { - file.size = size - await file.save() - v2Files.push(file._id.toString()) + const v2File = await FileModel.findOneAndUpdate( + { modelId, bucket, path }, + { + modelId, + name: `${version.version}-rawCodePath.zip`, + mime: 'application/x-zip-compressed', + bucket, + size, + path, + complete: true, + + createdAt: version.createdAt, + updatedAt: version.updatedAt, + }, + { + new: true, + upsert: true, // Make this update into an upsert + timestamps: false, + }, + ) + v2Files.push(v2File._id.toString()) } } if (version.files.rawDockerPath) { - const file = await new FileModel({ - modelId, - name: `${version.version}-rawDockerPath`, - mime: 'application/octet-stream', - bucket, - path: version.files.rawDockerPath, - complete: true, - }) - const size = await (await headObject(bucket, file.path)).ContentLength + const path = version.files.rawDockerPath + const size = await getObjectContentLength(bucket, path) if (size) { - file.size = size - await file.save() - v2Files.push(file._id.toString()) + const v2File = await FileModel.findOneAndUpdate( + { modelId, bucket, path }, + { + modelId, + name: `${version.version}-rawDockerPath.tar`, + mime: 'application/octet-stream', + bucket, + size, + path, + complete: true, + + createdAt: version.createdAt, + updatedAt: version.updatedAt, + }, + { + new: true, + upsert: true, // Make this update into an upsert + timestamps: false, + }, + ) + v2Files.push(v2File._id.toString()) } } From b05c450aeddf60d6501f20bd7254d75bfafa85b4 Mon Sep 17 00:00:00 2001 From: JR40159 <126243293+JR40159@users.noreply.github.com> Date: Fri, 12 Jan 2024 14:59:31 +0000 Subject: [PATCH 4/7] Migrate version approvals --- backend/src/scripts/migrateV2.ts | 61 ++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/backend/src/scripts/migrateV2.ts b/backend/src/scripts/migrateV2.ts index acf0cf7c9..99f82b973 100644 --- a/backend/src/scripts/migrateV2.ts +++ b/backend/src/scripts/migrateV2.ts @@ -1,16 +1,20 @@ import { uniqWith } from 'lodash-es' import { headObject, isNoSuchKeyException } from '../clients/s3.js' +import ApprovalModel from '../models/Approval.js' import DeploymentModelV1 from '../models/Deployment.js' import ModelModelV1 from '../models/Model.js' import FileModel from '../models/v2/File.js' import ModelModelV2, { CollaboratorEntry, ModelVisibility } from '../models/v2/Model.js' import ModelCardRevisionV2 from '../models/v2/ModelCardRevision.js' import Release from '../models/v2/Release.js' +import ReviewModel, { Decision, ReviewResponse } from '../models/v2/Review.js' import VersionModelV1 from '../models/Version.js' -import { VersionDoc } from '../types/types.js' +import { ApprovalCategory, ApprovalStates, ApprovalTypes, VersionDoc } from '../types/types.js' +import { ReviewKind } from '../types/v2/enums.js' import config from '../utils/config.js' import { connectToMongoose, disconnectFromMongoose } from '../utils/database.js' +import { toEntity } from '../utils/v2/entity.js' const MODEL_SCHEMA_MAP = { '/Minimal/General/v10': { @@ -255,13 +259,14 @@ async function migrateModel(modelId: string) { } } + const semver = `0.0.${i + 1}` await Release.findOneAndUpdate( - { modelId, semver: `0.0.${i + 1}` }, + { modelId, semver }, { modelId, modelCardVersion: i, - semver: `0.0.${i + 1}`, + semver, notes: `Migrated from V1. Orginal version ID: ${version.version}`, minor: false, @@ -282,6 +287,56 @@ async function migrateModel(modelId: string) { timestamps: false, }, ) + + const versionApprovals = await ApprovalModel.find({ + version: version._id, + approvalCategory: ApprovalCategory.Upload, + }).sort({ createdAt: 1 }) + + for (const approval of versionApprovals) { + let role + if (approval.approvalType === ApprovalTypes.Manager) { + role = 'msro' + } else if (approval.approvalType === ApprovalTypes.Reviewer) { + role = 'mtr' + } + + const responses: ReviewResponse[] = [] + for (let i = 0; i < approval.approvers.length; i++) { + const approver = approval.approvers[i] + if (approval.status === ApprovalStates.Accepted) { + responses.push({ + user: toEntity('user', approver.id), + decision: Decision.Approve, + }) + } else if (approval.status === ApprovalStates.Declined) { + responses.push({ + user: toEntity('user', approver.id), + decision: Decision.RequestChanges, + }) + } + } + await ReviewModel.findOneAndUpdate( + { modelId, semver, role }, + { + semver, + modelId, + + kind: ReviewKind.Release, + role, + + responses, + + createdAt: version.createdAt, + updatedAt: version.updatedAt, + }, + { + new: true, + upsert: true, // Make this update into an upsert + timestamps: false, + }, + ) + } } modelV2.card = { From 30ad361adab0e207815e5f54be9edfcd9bde0590 Mon Sep 17 00:00:00 2001 From: JR40159 <126243293+JR40159@users.noreply.github.com> Date: Mon, 15 Jan 2024 10:57:51 +0000 Subject: [PATCH 5/7] Add review comment --- backend/src/scripts/migrateV2.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/src/scripts/migrateV2.ts b/backend/src/scripts/migrateV2.ts index 99f82b973..71ed29a96 100644 --- a/backend/src/scripts/migrateV2.ts +++ b/backend/src/scripts/migrateV2.ts @@ -308,11 +308,13 @@ async function migrateModel(modelId: string) { responses.push({ user: toEntity('user', approver.id), decision: Decision.Approve, + comment: `Migrated from V1. Overall V1 approval decision to V2 individual user responses for the given role.`, }) } else if (approval.status === ApprovalStates.Declined) { responses.push({ user: toEntity('user', approver.id), decision: Decision.RequestChanges, + comment: `Migrated from V1. Overall V1 approval decision to V2 individual user responses for the given role.`, }) } } From bd3fe3c2f2f63a9b5102209a5460d5ddae2f1ab9 Mon Sep 17 00:00:00 2001 From: JR40159 <126243293+JR40159@users.noreply.github.com> Date: Mon, 15 Jan 2024 13:24:39 +0000 Subject: [PATCH 6/7] Add timestamps to responses --- backend/src/scripts/migrateV2.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/backend/src/scripts/migrateV2.ts b/backend/src/scripts/migrateV2.ts index 71ed29a96..96bef9d03 100644 --- a/backend/src/scripts/migrateV2.ts +++ b/backend/src/scripts/migrateV2.ts @@ -309,12 +309,16 @@ async function migrateModel(modelId: string) { user: toEntity('user', approver.id), decision: Decision.Approve, comment: `Migrated from V1. Overall V1 approval decision to V2 individual user responses for the given role.`, + createdAt: approval.createdAt, + updatedAt: approval.updatedAt, }) } else if (approval.status === ApprovalStates.Declined) { responses.push({ user: toEntity('user', approver.id), decision: Decision.RequestChanges, comment: `Migrated from V1. Overall V1 approval decision to V2 individual user responses for the given role.`, + createdAt: approval.createdAt, + updatedAt: approval.updatedAt, }) } } @@ -329,8 +333,8 @@ async function migrateModel(modelId: string) { responses, - createdAt: version.createdAt, - updatedAt: version.updatedAt, + createdAt: approval.createdAt, + updatedAt: approval.updatedAt, }, { new: true, From b1159463afb3b7a7a651b2c212dbdd8bf95b0be8 Mon Sep 17 00:00:00 2001 From: JR40159 <126243293+JR40159@users.noreply.github.com> Date: Mon, 15 Jan 2024 13:30:23 +0000 Subject: [PATCH 7/7] Fix import order --- backend/src/services/v2/release.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/services/v2/release.ts b/backend/src/services/v2/release.ts index 855a3feb1..478616edd 100644 --- a/backend/src/services/v2/release.ts +++ b/backend/src/services/v2/release.ts @@ -6,8 +6,8 @@ import { FileInterface } from '../../models/v2/File.js' import { ModelDoc, ModelInterface } from '../../models/v2/Model.js' import Release, { ImageRef, ReleaseDoc, ReleaseInterface } from '../../models/v2/Release.js' import { UserDoc } from '../../models/v2/User.js' -import { findDuplicates } from '../../utils/v2/array.js' import { WebhookEvent } from '../../models/v2/Webhook.js' +import { findDuplicates } from '../../utils/v2/array.js' import { BadReq, Forbidden, NotFound } from '../../utils/v2/error.js' import { isMongoServerError } from '../../utils/v2/mongo.js' import { getFileById, getFilesByIds } from './file.js'