From 9a464befa8434dfa806903174d954fef74c3ea10 Mon Sep 17 00:00:00 2001 From: P20179 Date: Mon, 17 Jul 2023 16:26:30 +0000 Subject: [PATCH 1/5] BAI-650 Added basic functionality for the check of file sizes --- frontend/src/Form/RenderFileTab.tsx | 89 +++++++++++++++++++---------- 1 file changed, 60 insertions(+), 29 deletions(-) diff --git a/frontend/src/Form/RenderFileTab.tsx b/frontend/src/Form/RenderFileTab.tsx index 75054085e..3531434a2 100644 --- a/frontend/src/Form/RenderFileTab.tsx +++ b/frontend/src/Form/RenderFileTab.tsx @@ -1,18 +1,36 @@ +import { Paper } from '@mui/material' import Box from '@mui/material/Box' import Divider from '@mui/material/Divider' import Grid from '@mui/material/Grid' import Stack from '@mui/material/Stack' import Typography from '@mui/material/Typography' -import React, { ChangeEvent, useMemo } from 'react' +import React, { ChangeEvent, useEffect, useMemo, useState } from 'react' import { RenderInterface, Step } from '../../types/interfaces' import { ModelUploadType } from '../../types/types' import { setStepState } from '../../utils/formUtils' import FileInput from '../common/FileInput' +import SubmissionError from './SubmissionError' export default function RenderFileTab({ step, splitSchema, setSplitSchema }: RenderInterface) { const { state } = step const { binary, code, docker } = state + const [error, setError] = useState(undefined) + const [totalFileSize, setTotalFileSize] = useState(0) + + useEffect(() => { + const codeSize = state.code ? state.code.size : 0 + const binarySize = state.binary ? state.binary.size : 0 + const dockerSize = state.docker ? state.docker.size : 0 + setTotalFileSize(codeSize + binarySize + dockerSize) + }, []) + + useEffect(() => { + setError(undefined) + //change in config + if (totalFileSize / 1024 > 1.9) setError('Model size exceeds maximum upload size') + console.log(totalFileSize) + }, [totalFileSize]) const buildOptionsStep = useMemo( () => splitSchema.steps.find((buildOptionSchemaStep) => buildOptionSchemaStep.section === 'buildOptions'), @@ -20,44 +38,57 @@ export default function RenderFileTab({ step, splitSchema, setSplitSchema }: Ren ) const handleCodeChange = (event: ChangeEvent) => { - if (event.target.files) setStepState(splitSchema, setSplitSchema, step, { ...state, code: event.target.files[0] }) + if (event.target.files) { + setStepState(splitSchema, setSplitSchema, step, { ...state, code: event.target.files[0] }) + setTotalFileSize(totalFileSize + event.target.files[0].size) + } } const handleBinaryChange = (event: ChangeEvent) => { - if (event.target.files) setStepState(splitSchema, setSplitSchema, step, { ...state, binary: event.target.files[0] }) + if (event.target.files) { + setStepState(splitSchema, setSplitSchema, step, { ...state, binary: event.target.files[0] }) + setTotalFileSize(totalFileSize + event.target.files[0].size) + } } const handleDockerChange = (event: ChangeEvent) => { - if (event.target.files) setStepState(splitSchema, setSplitSchema, step, { ...state, docker: event.target.files[0] }) + if (event.target.files) { + setStepState(splitSchema, setSplitSchema, step, { ...state, docker: event.target.files[0] }) + setTotalFileSize(totalFileSize + event.target.files[0].size) + } } return ( - - {buildOptionsStep !== undefined && buildOptionsStep.state.uploadType === ModelUploadType.Zip && ( - - - Upload a code file (.zip) - - - + + + {buildOptionsStep !== undefined && buildOptionsStep.state.uploadType === ModelUploadType.Zip && ( + + + Upload a code file (.zip) + + + + + Upload a binary file (.zip) + + + + )} + + {buildOptionsStep !== undefined && buildOptionsStep.state.uploadType === ModelUploadType.ModelCard && ( + Uploading a model card without any code or binary files + )} + {buildOptionsStep !== undefined && buildOptionsStep.state.uploadType === ModelUploadType.Docker && ( - Upload a binary file (.zip) - + + Upload a docker file (.tar) + + - - )} - {buildOptionsStep !== undefined && buildOptionsStep.state.uploadType === ModelUploadType.ModelCard && ( - Uploading a model card without any code or binary files - )} - {buildOptionsStep !== undefined && buildOptionsStep.state.uploadType === ModelUploadType.Docker && ( - - - Upload a docker file (.tar) - - - - )} - + )} + + + ) } @@ -76,7 +107,7 @@ export function fileTabComplete(step: Step) { case ModelUploadType.ModelCard: return true case ModelUploadType.Zip: - return step.state.binary && step.state.code + return step.state.binary && step.state.code && step.state.binary.size / 1024 + step.state.code.size / 1024 <= 1.9 case ModelUploadType.Docker: return !!step.state.docker default: From cc539473354ad0eff87f8aa4d3cdf225684f9809 Mon Sep 17 00:00:00 2001 From: P20179 Date: Tue, 18 Jul 2023 15:45:41 +0000 Subject: [PATCH 2/5] BAI-650 Updated functionality to persist the error message if user browses to different pages, handle the errors, prevent user from submitting the model if exceeds max, add max to config so it is configurable --- backend/config/default.cjs | 1 + backend/src/utils/config.ts | 2 ++ frontend/pages/model/[uuid]/new-version.tsx | 12 +++++--- frontend/pages/upload.tsx | 20 ++++++------ frontend/src/Form/RenderFileTab.tsx | 34 ++++++++++++++------- frontend/src/common/Loading.tsx | 10 ++++++ frontend/types/types.ts | 3 ++ frontend/utils/byteUtils.ts | 3 ++ 8 files changed, 60 insertions(+), 25 deletions(-) create mode 100644 frontend/src/common/Loading.tsx create mode 100644 frontend/utils/byteUtils.ts diff --git a/backend/config/default.cjs b/backend/config/default.cjs index 8ba6fc89d..54b3877f0 100644 --- a/backend/config/default.cjs +++ b/backend/config/default.cjs @@ -215,5 +215,6 @@ module.exports = { image: 'seldonio/seldon-core-s2i-python37:1.10.0', }, ], + maxModelSizeGB: 50 }, } diff --git a/backend/src/utils/config.ts b/backend/src/utils/config.ts index 0a22cfb25..d7d463c08 100644 --- a/backend/src/utils/config.ts +++ b/backend/src/utils/config.ts @@ -147,6 +147,8 @@ export interface Config { name: string image: string }> + //max model size is calculated in gigabytes + maxModelSizeGB: number } } diff --git a/frontend/pages/model/[uuid]/new-version.tsx b/frontend/pages/model/[uuid]/new-version.tsx index 6c256708b..2db33d895 100644 --- a/frontend/pages/model/[uuid]/new-version.tsx +++ b/frontend/pages/model/[uuid]/new-version.tsx @@ -1,7 +1,9 @@ import Paper from '@mui/material/Paper' import axios from 'axios' +import { useGetUiConfig } from 'data/uiConfig' import { useRouter } from 'next/router' import React, { useEffect, useState } from 'react' +import Loading from 'src/common/Loading' import MessageAlert from 'src/MessageAlert' import { useGetModel, useGetModelVersions } from '../../../data/model' @@ -47,6 +49,7 @@ function Upload() { const { model, isModelLoading, isModelError, mutateModel } = useGetModel(modelUuid) const { schema, isSchemaLoading, isSchemaError } = useGetSchema(model?.schemaRef) const { versions } = useGetModelVersions(modelUuid) + const { uiConfig, isUiConfigError, isUiConfigLoading } = useGetUiConfig() const cModel = useCacheVariable(model) const cSchema = useCacheVariable(schema) @@ -88,7 +91,7 @@ function Upload() { render: RenderFileTab, renderBasic: RenderBasicFileTab, - isComplete: fileTabComplete, + isComplete: (step) => fileTabComplete(step, uiConfig ? uiConfig.maxModelSizeGB : 0), }) ) @@ -115,16 +118,17 @@ function Upload() { } setSplitSchema({ reference: cSchema.reference, steps }) - }, [cModel, cSchema]) + }, [cModel, cSchema, uiConfig]) const errorWrapper = MultipleErrorWrapper(`Unable to load edit page`, { isModelError, isSchemaError, + isUiConfigError, }) if (errorWrapper) return errorWrapper - if (isModelLoading || isSchemaLoading) { - return null + if (isModelLoading || isSchemaLoading || isUiConfigLoading) { + return } if (!model || !schema) { diff --git a/frontend/pages/upload.tsx b/frontend/pages/upload.tsx index 61016db9f..157695b69 100644 --- a/frontend/pages/upload.tsx +++ b/frontend/pages/upload.tsx @@ -2,8 +2,10 @@ import Box from '@mui/material/Box' import Grid from '@mui/material/Grid' import Paper from '@mui/material/Paper' import axios from 'axios' +import { useGetUiConfig } from 'data/uiConfig' import { useRouter } from 'next/router' import React, { useEffect, useState } from 'react' +import Loading from 'src/common/Loading' import { useGetDefaultSchema, useGetSchemas } from '../data/schema' import { useGetCurrentUser } from '../data/user' @@ -47,6 +49,7 @@ function Upload() { const { defaultSchema, isDefaultSchemaError, isDefaultSchemaLoading } = useGetDefaultSchema('UPLOAD') const { schemas, isSchemasLoading, isSchemasError } = useGetSchemas('UPLOAD') const { currentUser, isCurrentUserLoading, isCurrentUserError } = useGetCurrentUser() + const { uiConfig, isUiConfigError, isUiConfigLoading } = useGetUiConfig() const router = useRouter() @@ -104,7 +107,7 @@ function Upload() { render: RenderFileTab, renderBasic: RenderBasicFileTab, - isComplete: fileTabComplete, + isComplete: (step) => fileTabComplete(step, uiConfig ? uiConfig.maxModelSizeGB : 0), }) ) @@ -131,7 +134,7 @@ function Upload() { } setSplitSchema({ reference, steps }) - }, [currentSchema, user]) + }, [currentSchema, uiConfig, user]) const errorWrapper = MultipleErrorWrapper( `Unable to load upload page`, @@ -139,19 +142,16 @@ function Upload() { isDefaultSchemaError, isSchemasError, isCurrentUserError, + isUiConfigError, }, MinimalErrorWrapper ) if (errorWrapper) return errorWrapper - if (isDefaultSchemaLoading || isSchemasLoading || isCurrentUserLoading) { - return null - } - const Loading = - - if (isDefaultSchemaLoading || !defaultSchema) return Loading - if (isSchemasLoading || !schemas) return Loading - if (isCurrentUserLoading || !currentUser) return Loading + if (isDefaultSchemaLoading || !defaultSchema) return + if (isSchemasLoading || !schemas) return + if (isCurrentUserLoading || !currentUser) return + if (isUiConfigLoading || !uiConfig) return const onSubmit = async () => { setError(undefined) diff --git a/frontend/src/Form/RenderFileTab.tsx b/frontend/src/Form/RenderFileTab.tsx index 3531434a2..bda816133 100644 --- a/frontend/src/Form/RenderFileTab.tsx +++ b/frontend/src/Form/RenderFileTab.tsx @@ -4,10 +4,13 @@ import Divider from '@mui/material/Divider' import Grid from '@mui/material/Grid' import Stack from '@mui/material/Stack' import Typography from '@mui/material/Typography' -import React, { ChangeEvent, useEffect, useMemo, useState } from 'react' +import { useGetUiConfig } from 'data/uiConfig' +import { ChangeEvent, useEffect, useMemo, useState } from 'react' +import Loading from 'src/common/Loading' +import { convertGigabytesToBytes } from 'utils/byteUtils' import { RenderInterface, Step } from '../../types/interfaces' -import { ModelUploadType } from '../../types/types' +import { ModelUploadType, UiConfig } from '../../types/types' import { setStepState } from '../../utils/formUtils' import FileInput from '../common/FileInput' import SubmissionError from './SubmissionError' @@ -15,22 +18,25 @@ import SubmissionError from './SubmissionError' export default function RenderFileTab({ step, splitSchema, setSplitSchema }: RenderInterface) { const { state } = step const { binary, code, docker } = state - const [error, setError] = useState(undefined) + const [error, setError] = useState('') const [totalFileSize, setTotalFileSize] = useState(0) + const { uiConfig, isUiConfigError, isUiConfigLoading } = useGetUiConfig() + useEffect(() => { const codeSize = state.code ? state.code.size : 0 const binarySize = state.binary ? state.binary.size : 0 const dockerSize = state.docker ? state.docker.size : 0 setTotalFileSize(codeSize + binarySize + dockerSize) - }, []) + }, [state.binary, state.code, state.docker]) useEffect(() => { - setError(undefined) - //change in config - if (totalFileSize / 1024 > 1.9) setError('Model size exceeds maximum upload size') - console.log(totalFileSize) - }, [totalFileSize]) + if (isUiConfigError) setError(isUiConfigError.message) + else if (!uiConfig) setError('Failed to load ui config') + else if (totalFileSize > convertGigabytesToBytes(uiConfig.maxModelSizeGB)) + setError('Model size exceeds maximum upload size of') + else setError('') + }, [isUiConfigError, totalFileSize, uiConfig]) const buildOptionsStep = useMemo( () => splitSchema.steps.find((buildOptionSchemaStep) => buildOptionSchemaStep.section === 'buildOptions'), @@ -58,6 +64,8 @@ export default function RenderFileTab({ step, splitSchema, setSplitSchema }: Ren } } + if (isUiConfigLoading) return + return ( @@ -92,9 +100,13 @@ export default function RenderFileTab({ step, splitSchema, setSplitSchema }: Ren ) } -export function fileTabComplete(step: Step) { +export function fileTabComplete(step: Step, maxModelSizeGB: UiConfig['maxModelSizeGB']) { if (!step.steps) return false + const totalFileSize = + (step.state.binary ? step.state.binary.size : 0) + + (step.state.code ? step.state.code.size : 0) + + (step.state.docker ? step.state.docker.size : 0) const buildOptionsStep = step.steps.find((buildOptionSchemaStep) => buildOptionSchemaStep.section === 'buildOptions') const hasUploadType = !!buildOptionsStep?.state?.uploadType @@ -107,7 +119,7 @@ export function fileTabComplete(step: Step) { case ModelUploadType.ModelCard: return true case ModelUploadType.Zip: - return step.state.binary && step.state.code && step.state.binary.size / 1024 + step.state.code.size / 1024 <= 1.9 + return step.state.binary && step.state.code && totalFileSize <= convertGigabytesToBytes(maxModelSizeGB) case ModelUploadType.Docker: return !!step.state.docker default: diff --git a/frontend/src/common/Loading.tsx b/frontend/src/common/Loading.tsx new file mode 100644 index 000000000..0d28a1004 --- /dev/null +++ b/frontend/src/common/Loading.tsx @@ -0,0 +1,10 @@ +import { Box, CircularProgress } from '@mui/material' +import { ReactElement } from 'react' + +export default function Loading(): ReactElement { + return ( + + + + ) +} diff --git a/frontend/types/types.ts b/frontend/types/types.ts index 4240f1527..eb70fcf13 100644 --- a/frontend/types/types.ts +++ b/frontend/types/types.ts @@ -137,6 +137,9 @@ export interface UiConfig { } seldonVersions: Array + + //max model size is calculated in gigabytes + maxModelSizeGB: number } export type SeldonVersion = { diff --git a/frontend/utils/byteUtils.ts b/frontend/utils/byteUtils.ts new file mode 100644 index 000000000..930385581 --- /dev/null +++ b/frontend/utils/byteUtils.ts @@ -0,0 +1,3 @@ +export function convertGigabytesToBytes(gigabytes: number) { + return gigabytes * 1024 * 1024 * 1024 +} From fbb8d5f13d5f298b3d74a3f2a9007a2ef668e4c3 Mon Sep 17 00:00:00 2001 From: P20179 Date: Wed, 19 Jul 2023 13:58:24 +0000 Subject: [PATCH 3/5] BAI-650 Fixed test failures --- backend/src/routes/v1/specification.ts | 6 ++++++ backend/src/types/types.ts | 1 + 2 files changed, 7 insertions(+) diff --git a/backend/src/routes/v1/specification.ts b/backend/src/routes/v1/specification.ts index 8af7f8477..66199c385 100644 --- a/backend/src/routes/v1/specification.ts +++ b/backend/src/routes/v1/specification.ts @@ -507,6 +507,12 @@ function parseValue(value: unknown) { example: value, } } + if (typeof value === 'number') { + return { + type: 'number', + example: value, + } + } if (typeof value === 'boolean') { return { type: 'boolean', diff --git a/backend/src/types/types.ts b/backend/src/types/types.ts index d18f35cad..988432698 100644 --- a/backend/src/types/types.ts +++ b/backend/src/types/types.ts @@ -133,6 +133,7 @@ export interface UiConfig { } seldonVersions: Array + maxModelSizeGB: number } export type SeldonVersion = { From a6d1d479deb3232b50bfb9b053651b7d902fc8de Mon Sep 17 00:00:00 2001 From: P20179 Date: Wed, 19 Jul 2023 14:08:03 +0000 Subject: [PATCH 4/5] BAI-650 Added a more complete error message that tells user the max upload size --- frontend/src/Form/RenderFileTab.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/Form/RenderFileTab.tsx b/frontend/src/Form/RenderFileTab.tsx index bda816133..b6fdc9a99 100644 --- a/frontend/src/Form/RenderFileTab.tsx +++ b/frontend/src/Form/RenderFileTab.tsx @@ -34,7 +34,7 @@ export default function RenderFileTab({ step, splitSchema, setSplitSchema }: Ren if (isUiConfigError) setError(isUiConfigError.message) else if (!uiConfig) setError('Failed to load ui config') else if (totalFileSize > convertGigabytesToBytes(uiConfig.maxModelSizeGB)) - setError('Model size exceeds maximum upload size of') + setError('Model size exceeds maximum upload size of ' + uiConfig.maxModelSizeGB + 'GB') else setError('') }, [isUiConfigError, totalFileSize, uiConfig]) From 8587d35edef4d289631f9467bb69764e8c768f65 Mon Sep 17 00:00:00 2001 From: P20179 Date: Fri, 21 Jul 2023 12:50:57 +0000 Subject: [PATCH 5/5] BAI-650 Slight edit to error message --- frontend/src/Form/RenderFileTab.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/Form/RenderFileTab.tsx b/frontend/src/Form/RenderFileTab.tsx index b6fdc9a99..610005ba5 100644 --- a/frontend/src/Form/RenderFileTab.tsx +++ b/frontend/src/Form/RenderFileTab.tsx @@ -32,7 +32,7 @@ export default function RenderFileTab({ step, splitSchema, setSplitSchema }: Ren useEffect(() => { if (isUiConfigError) setError(isUiConfigError.message) - else if (!uiConfig) setError('Failed to load ui config') + else if (!uiConfig) setError('Failed to load UI config') else if (totalFileSize > convertGigabytesToBytes(uiConfig.maxModelSizeGB)) setError('Model size exceeds maximum upload size of ' + uiConfig.maxModelSizeGB + 'GB') else setError('')