Feature/migrate release and review

backend/config/default.cjs

@@ -40,7 +40,7 @@ module.exports = {
   mongo: {
     // A mongo connection URI, can contain usernames, passwords, replica set information, etc.
     // See: https://www.mongodb.com/docs/manual/reference/connection-string/
-    uri: 'mongodb://localhost:27017/bailo',
+    uri: 'mongodb://localhost:27017/bailo?directConnection=true',
   },
 
   minio: {

backend/src/clients/s3.ts

@@ -1,4 +1,4 @@
-import { GetObjectCommand, GetObjectRequest, S3Client } from '@aws-sdk/client-s3'
+import { GetObjectCommand, GetObjectRequest, HeadObjectRequest, NoSuchKey, S3Client } from '@aws-sdk/client-s3'
 import { Upload } from '@aws-sdk/lib-storage'
 import { NodeHttpHandler } from '@smithy/node-http-handler'
 
@@ -57,3 +57,21 @@ export async function getObjectStream(bucket: string, key: string, range?: { sta
 
   return response
 }
+
+export async function headObject(bucket: string, key: string) {
+  const client = await getS3Client()
+
+  const input: HeadObjectRequest = {
+    Bucket: bucket,
+    Key: key,
+  }
+
+  const command = new GetObjectCommand(input)
+  const response = await client.send(command)
+
+  return response
+}
+
+export function isNoSuchKeyException(err: any): err is NoSuchKey {
+  return err?.name === 'NoSuchKey'
+}

backend/src/scripts/migrateV2.ts

@@ -1,12 +1,20 @@
 import { uniqWith } from 'lodash-es'
 
+import { headObject, isNoSuchKeyException } from '../clients/s3.js'
+import ApprovalModel from '../models/Approval.js'
 import DeploymentModelV1 from '../models/Deployment.js'
 import ModelModelV1 from '../models/Model.js'
+import FileModel from '../models/v2/File.js'
 import ModelModelV2, { CollaboratorEntry, ModelVisibility } from '../models/v2/Model.js'
 import ModelCardRevisionV2 from '../models/v2/ModelCardRevision.js'
+import Release from '../models/v2/Release.js'
+import ReviewModel, { Decision, ReviewResponse } from '../models/v2/Review.js'
 import VersionModelV1 from '../models/Version.js'
-import { VersionDoc } from '../types/types.js'
+import { ApprovalCategory, ApprovalStates, ApprovalTypes, VersionDoc } from '../types/types.js'
+import { ReviewKind } from '../types/v2/enums.js'
+import config from '../utils/config.js'
 import { connectToMongoose, disconnectFromMongoose } from '../utils/database.js'
+import { toEntity } from '../utils/v2/entity.js'
 
 const MODEL_SCHEMA_MAP = {
   '/Minimal/General/v10': {
@@ -37,6 +45,19 @@ const _DEPLOYMENT_SCHEMA_MAP = {
   },
 }
 
+async function getObjectContentLength(bucket: string, object: string) {
+  let objectMeta
+  try {
+    objectMeta = await headObject(bucket, object)
+  } catch (e) {
+    if (isNoSuchKeyException(e)) {
+      return
+    }
+    throw e
+  }
+  return objectMeta.ContentLength
+}
+
 function identityConversion(old: string) {
   return old
 }
@@ -49,7 +70,7 @@ export async function migrateAllModels() {
 }
 
 async function migrateModel(modelId: string) {
-  const model = await ModelModelV1.findOne({ uuid: modelId }).populate('latestVersion')
+  const model = await ModelModelV1.findOne({ _id: modelId }).populate('latestVersion')
   if (!model) throw new Error(`Model not found: ${modelId}`)
   model.latestVersion = model.latestVersion as VersionDoc
 
@@ -153,6 +174,175 @@ async function migrateModel(modelId: string) {
         timestamps: false,
       },
     )
+
+    const v2Files: string[] = []
+    const bucket = config.minio.buckets.uploads
+    if (version.files.rawBinaryPath) {
+      const path = version.files.rawBinaryPath
+      const size = await getObjectContentLength(bucket, path)
+      if (size) {
+        const v2File = await FileModel.findOneAndUpdate(
+          { modelId, bucket, path },
+          {
+            modelId,
+            name: `${version.version}-rawBinaryPath.zip`,
+            mime: 'application/x-zip-compressed',
+            size,
+            bucket,
+            path,
+            complete: true,
+
+            createdAt: version.createdAt,
+            updatedAt: version.updatedAt,
+          },
+          {
+            new: true,
+            upsert: true, // Make this update into an upsert
+            timestamps: false,
+          },
+        )
+        v2Files.push(v2File._id.toString())
+      }
+    }
+    if (version.files.rawCodePath) {
+      const path = version.files.rawCodePath
+      const size = await getObjectContentLength(bucket, path)
+      if (size) {
+        const v2File = await FileModel.findOneAndUpdate(
+          { modelId, bucket, path },
+          {
+            modelId,
+            name: `${version.version}-rawCodePath.zip`,
+            mime: 'application/x-zip-compressed',
+            bucket,
+            size,
+            path,
+            complete: true,
+
+            createdAt: version.createdAt,
+            updatedAt: version.updatedAt,
+          },
+          {
+            new: true,
+            upsert: true, // Make this update into an upsert
+            timestamps: false,
+          },
+        )
+        v2Files.push(v2File._id.toString())
+      }
+    }
+    if (version.files.rawDockerPath) {
+      const path = version.files.rawDockerPath
+      const size = await getObjectContentLength(bucket, path)
+      if (size) {
+        const v2File = await FileModel.findOneAndUpdate(
+          { modelId, bucket, path },
+          {
+            modelId,
+            name: `${version.version}-rawDockerPath.tar`,
+            mime: 'application/octet-stream',
+            bucket,
+            size,
+            path,
+            complete: true,
+
+            createdAt: version.createdAt,
+            updatedAt: version.updatedAt,
+          },
+          {
+            new: true,
+            upsert: true, // Make this update into an upsert
+            timestamps: false,
+          },
+        )
+        v2Files.push(v2File._id.toString())
+      }
+    }
+
+    const semver = `0.0.${i + 1}`
+    await Release.findOneAndUpdate(
+      { modelId, semver },
+      {
+        modelId,
+        modelCardVersion: i,
+
+        semver,
+        notes: `Migrated from V1. Orginal version ID: ${version.version}`,
+
+        minor: false,
+        draft: false,
+
+        fileIds: v2Files,
+        // Not sure about images
+        images: [],
+
+        createdBy: 'system',
+        createdAt: version.createdAt,
+        updatedAt: version.updatedAt,
+      },
+
+      {
+        new: true,
+        upsert: true, // Make this update into an upsert
+        timestamps: false,
+      },
+    )
+
+    const versionApprovals = await ApprovalModel.find({
+      version: version._id,
+      approvalCategory: ApprovalCategory.Upload,
+    }).sort({ createdAt: 1 })
+
+    for (const approval of versionApprovals) {
+      let role
+      if (approval.approvalType === ApprovalTypes.Manager) {
+        role = 'msro'
+      } else if (approval.approvalType === ApprovalTypes.Reviewer) {
+        role = 'mtr'
+      }
+
+      const responses: ReviewResponse[] = []
+      for (let i = 0; i < approval.approvers.length; i++) {
+        const approver = approval.approvers[i]
+        if (approval.status === ApprovalStates.Accepted) {
+          responses.push({
+            user: toEntity('user', approver.id),
+            decision: Decision.Approve,
+            comment: `Migrated from V1. Overall V1 approval decision to V2 individual user responses for the given role.`,
+            createdAt: approval.createdAt,
+            updatedAt: approval.updatedAt,
+          })
+        } else if (approval.status === ApprovalStates.Declined) {
+          responses.push({
+            user: toEntity('user', approver.id),
+            decision: Decision.RequestChanges,
+            comment: `Migrated from V1. Overall V1 approval decision to V2 individual user responses for the given role.`,
+            createdAt: approval.createdAt,
+            updatedAt: approval.updatedAt,
+          })
+        }
+      }
+      await ReviewModel.findOneAndUpdate(
+        { modelId, semver, role },
+        {
+          semver,
+          modelId,
+
+          kind: ReviewKind.Release,
+          role,
+
+          responses,
+
+          createdAt: approval.createdAt,
+          updatedAt: approval.updatedAt,
+        },
+        {
+          new: true,
+          upsert: true, // Make this update into an upsert
+          timestamps: false,
+        },
+      )
+    }
   }
 
   modelV2.card = {
@@ -171,7 +361,7 @@ async function migrateModel(modelId: string) {
 
 await connectToMongoose()
 
-// await migrateAllModels()
-await migrateModel('minimal-model-for-testing-im7q59')
+await migrateAllModels()
+//await migrateModel('minimal-model-for-testing-im7q59')
 
 setTimeout(disconnectFromMongoose, 500)

backend/src/services/v2/release.ts

@@ -6,8 +6,8 @@ import { FileInterface } from '../../models/v2/File.js'
 import { ModelDoc, ModelInterface } from '../../models/v2/Model.js'
 import Release, { ImageRef, ReleaseDoc, ReleaseInterface } from '../../models/v2/Release.js'
 import { UserDoc } from '../../models/v2/User.js'
-import { findDuplicates } from '../../utils/v2/array.js'
 import { WebhookEvent } from '../../models/v2/Webhook.js'
+import { findDuplicates } from '../../utils/v2/array.js'
 import { BadReq, Forbidden, NotFound } from '../../utils/v2/error.js'
 import { isMongoServerError } from '../../utils/v2/mongo.js'
 import { getFileById, getFilesByIds } from './file.js'

backend/test/clients/s3.spec.ts

@@ -1,6 +1,6 @@
 import { describe, expect, test, vi } from 'vitest'
 
-import { getObjectStream } from '../../src/clients/s3.js'
+import { getObjectStream, headObject } from '../../src/clients/s3.js'
 
 const s3Mocks = vi.hoisted(() => {
   const send = vi.fn(() => 'response')
@@ -29,4 +29,18 @@ describe('clients > s3', () => {
     expect(s3Mocks.send).toHaveBeenCalled()
     expect(response).toBe('response')
   })
+
+  test('headObject > success', async () => {
+    const bucket = 'test-bucket'
+    const key = 'test-key'
+
+    const response = await headObject(bucket, key)
+
+    expect(s3Mocks.GetObjectCommand).toHaveBeenCalledWith({
+      Bucket: bucket,
+      Key: key,
+    })
+    expect(s3Mocks.send).toHaveBeenCalled()
+    expect(response).toBe('response')
+  })
 })