feat: downloading blocks of a file by index (#271)

* feat: implemented uploading data by block index

* feat: tests and docs for uploading by block index

* feat: downloading data by index

* fix: added options

* docs: added example

* test: include more info about fairos error

* test: include more info about fairos error

* test: added configurable fairos image to prevent errors from non-stable version

* test: added configurable fairos image to prevent errors from non-stable version

.github/workflows/tests.yaml

@@ -10,6 +10,7 @@ on:
 
 env:
   BEE_VERSION: '1.11.1'
+  FAIROS_IMAGE: 'fairdatasociety/fairos-dfs:v0.9.6'
 
 jobs:
   nodejs:
@@ -38,7 +39,7 @@ jobs:
         run: npm install -g @fairdatasociety/fdp-play
 
       - name: Run fdp-play
-        run: fdp-play start -d --fairos --bee-version $BEE_VERSION
+        run: fdp-play start -d --fairos --fairos-image $FAIROS_IMAGE --bee-version $BEE_VERSION
 
       ## Try getting the node modules from cache, if failed npm ci
       - uses: actions/cache@v2

README.md

@@ -288,6 +288,14 @@ await fdp.file.downloadData('my-new-pod', '/myfile.txt', {
     console.log(event)
   }
 })
+
+// or you can download data block-by-block to combine it later
+const blockSize = 1000000
+const fileMeta = await fdp.file.getMetadata(pod, fullPath)
+const result = new Uint8Array(fileMeta.fileSize)
+for (let i = 0; i < blocksCount; i++) {
+  result.set(await fdp.file.downloadDataBlock(fileMeta, i), i * blockSize)
+}
 ```
 
 Deleting a pod

src/file/file.ts

@@ -12,11 +12,18 @@ import {
   uploadBytes,
 } from './utils'
 import { writeFeedData } from '../feed/api'
-import { downloadData, uploadData, uploadDataBlock } from './handler'
+import { downloadData, getFileMetadataWithBlocks, uploadData, uploadDataBlock } from './handler'
 import { getFileMetadataRawBytes, rawFileMetadataToFileMetadata } from './adapter'
-import { DataDownloadOptions, DataUploadOptions, ExternalDataBlock, FileReceiveOptions, FileShareInfo } from './types'
+import {
+  DataDownloadOptions,
+  DataUploadOptions,
+  ExternalDataBlock,
+  FileMetadataWithBlocks,
+  FileReceiveOptions,
+  FileShareInfo,
+} from './types'
 import { addEntryToDirectory, DEFAULT_UPLOAD_OPTIONS, removeEntryFromDirectory } from '../content-items/handler'
-import { Reference } from '@ethersphere/bee-js'
+import { BeeRequestOptions, Reference } from '@ethersphere/bee-js'
 import { getRawMetadata } from '../content-items/utils'
 import { assertRawFileMetadata, combine, splitPath } from '../directory/utils'
 import { assertEncryptedReference, EncryptedReference } from '../utils/hex'
@@ -64,6 +71,7 @@ export class File {
   ): Promise<FileMetadata> {
     options = { ...DEFAULT_UPLOAD_OPTIONS, ...options }
     assertAccount(this.accountData)
+    assertPodName(podName)
 
     return uploadData(podName, fullPath, data, this.accountData, options)
   }
@@ -166,4 +174,51 @@ export class File {
       index: blockIndex,
     }
   }
+
+  /**
+   * Downloads file metadata with blocks data
+   *
+   * @param podName pod where file is stored
+   * @param fullPath full path of the file
+   * @param downloadOptions bee download options
+   * @param options data download options
+   */
+  async getMetadata(
+    podName: string,
+    fullPath: string,
+    downloadOptions?: BeeRequestOptions,
+    options?: DataDownloadOptions,
+  ): Promise<FileMetadataWithBlocks> {
+    assertAccount(this.accountData)
+    assertPodName(podName)
+    assertFullPathWithName(fullPath)
+
+    return getFileMetadataWithBlocks(
+      this.accountData.connection.bee,
+      this.accountData,
+      podName,
+      fullPath,
+      downloadOptions,
+      options,
+    )
+  }
+
+  /**
+   * Downloads data block using file metadata
+   *
+   * @param meta file metadata
+   * @param blockIndex block index
+   * @param downloadOptions bee download options
+   */
+  async downloadDataBlock(
+    meta: FileMetadataWithBlocks,
+    blockIndex: number,
+    downloadOptions?: BeeRequestOptions,
+  ): Promise<Uint8Array> {
+    if (blockIndex < 0 || blockIndex >= meta.blocks.length) {
+      throw new Error('"blockIndex" is out of bounds')
+    }
+
+    return this.accountData.connection.bee.downloadData(meta.blocks[blockIndex].reference, downloadOptions)
+  }
 }

src/file/handler.ts

@@ -8,7 +8,8 @@ import {
   DEFAULT_FILE_PERMISSIONS,
   downloadBlocksManifest,
   externalDataBlocksToBlocks,
-  extractPathInfo, getDataBlock,
+  extractPathInfo,
+  getDataBlock,
   getFileMode,
   isExternalDataBlocks,
   updateDownloadProgress,
@@ -27,6 +28,7 @@ import {
   DataUploadOptions,
   DownloadProgressType,
   ExternalDataBlock,
+  FileMetadataWithBlocks,
   UploadProgressType,
 } from './types'
 import { assertPodName, getExtendedPodsListByAccountData, META_VERSION } from '../pod/utils'
@@ -71,23 +73,24 @@ export async function getFileMetadata(
 }
 
 /**
- * Downloads file parts and compile them into Data
+ * Gets file metadata with blocks
  *
+ * @param bee Bee
  * @param accountData account data
  * @param podName pod name
  * @param fullPath full path to the file
  * @param downloadOptions download options
  * @param dataDownloadOptions data download options
  */
-export async function downloadData(
+export async function getFileMetadataWithBlocks(
+  bee: Bee,
   accountData: AccountData,
   podName: string,
   fullPath: string,
   downloadOptions?: BeeRequestOptions,
   dataDownloadOptions?: DataDownloadOptions,
-): Promise<Data> {
+): Promise<FileMetadataWithBlocks> {
   dataDownloadOptions = dataDownloadOptions ?? {}
-  const bee = accountData.connection.bee
   updateDownloadProgress(dataDownloadOptions, DownloadProgressType.GetPodInfo)
   const { podAddress, pod } = await getExtendedPodsListByAccountData(accountData, podName)
   updateDownloadProgress(dataDownloadOptions, DownloadProgressType.GetPathInfo)
@@ -101,15 +104,48 @@ export async function downloadData(
   updateDownloadProgress(dataDownloadOptions, DownloadProgressType.DownloadBlocksMeta)
   const blocks = await downloadBlocksManifest(bee, fileMetadata.blocksReference, downloadOptions)
 
+  return {
+    ...fileMetadata,
+    ...blocks,
+  }
+}
+
+/**
+ * Downloads file parts and compile them into Data
+ *
+ * @param accountData account data
+ * @param podName pod name
+ * @param fullPath full path to the file
+ * @param downloadOptions download options
+ * @param dataDownloadOptions data download options
+ */
+export async function downloadData(
+  accountData: AccountData,
+  podName: string,
+  fullPath: string,
+  downloadOptions?: BeeRequestOptions,
+  dataDownloadOptions?: DataDownloadOptions,
+): Promise<Data> {
+  dataDownloadOptions = dataDownloadOptions ?? {}
+  const bee = accountData.connection.bee
+  const { blocks } = await getFileMetadataWithBlocks(
+    bee,
+    accountData,
+    podName,
+    fullPath,
+    downloadOptions,
+    dataDownloadOptions,
+  )
+
   let totalLength = 0
-  for (const block of blocks.blocks) {
+  for (const block of blocks) {
     totalLength += block.size
   }
 
   const result = new Uint8Array(totalLength)
   let offset = 0
-  const totalBlocks = blocks.blocks.length
-  for (const [currentBlockId, block] of blocks.blocks.entries()) {
+  const totalBlocks = blocks.length
+  for (const [currentBlockId, block] of blocks.entries()) {
     const blockData = {
       totalBlocks,
       currentBlockId,

src/file/types.ts

@@ -1,5 +1,5 @@
 import { Reference } from '@ethersphere/bee-js'
-import { RawFileMetadata } from '../pod/types'
+import { FileMetadata, RawFileMetadata } from '../pod/types'
 
 /**
  * Download progress info
@@ -164,6 +164,11 @@ export interface Blocks {
   blocks: Block[]
 }
 
+/**
+ * FDP file metadata with blocks data
+ */
+export type FileMetadataWithBlocks = FileMetadata & Blocks
+
 /**
  * FDP file block format
  */

src/types.ts

@@ -13,6 +13,7 @@ export {
   DataDownloadOptions,
   Block,
   ExternalDataBlock,
+  FileMetadataWithBlocks,
 } from './file/types'
 export {
   calcUploadBlockPercentage,

test/integration/node/download-by-index.spec.ts

@@ -0,0 +1,24 @@
+import { createFdp, generateRandomHexString, generateUser, makeFileContent } from '../../utils'
+import { wrapBytesWithHelpers } from '../../../src/utils/bytes'
+
+jest.setTimeout(400000)
+it('Download by index', async () => {
+  const fdp = createFdp()
+  const pod = generateRandomHexString()
+  const { size, blocksCount, blockSize, content, filename, fullPath } = makeFileContent(5000005)
+  generateUser(fdp)
+  await fdp.personalStorage.create(pod)
+  await fdp.file.uploadData(pod, fullPath, content)
+  const fileMeta = await fdp.file.getMetadata(pod, fullPath)
+  expect(fileMeta.fileName).toEqual(filename)
+  expect(fileMeta.blockSize).toEqual(blockSize)
+  expect(fileMeta.fileSize).toEqual(size)
+
+  const result = new Uint8Array(fileMeta.fileSize)
+  for (let i = 0; i < blocksCount; i++) {
+    result.set(await fdp.file.downloadDataBlock(fileMeta, i), i * blockSize)
+  }
+
+  const dataBig = wrapBytesWithHelpers(await fdp.file.downloadData(pod, fullPath)).text()
+  expect(dataBig).toEqual(content)
+})

test/integration/node/upload-by-index.spec.ts

@@ -1,6 +1,6 @@
 import { createFdp, generateRandomHexString, generateUser, makeFileContent } from '../../utils'
 import { wrapBytesWithHelpers } from '../../../src/utils/bytes'
-import { getDataBlock } from '../../../src/file/utils'
+import { getDataBlock } from '../../../src'
 
 jest.setTimeout(400000)
 it('Upload by index', async () => {

test/utils/fairos-api.ts

@@ -37,13 +37,24 @@ export class FairOSApi {
 
       return config
     })
-    axios.interceptors.response.use(response => {
-      if (this.withCredentials && response.headers['set-cookie']) {
-        this.cookies = response.headers['set-cookie']?.[0] as string
-      }
+    axios.interceptors.response.use(
+      response => {
+        if (this.withCredentials && response.headers['set-cookie']) {
+          this.cookies = response.headers['set-cookie']?.[0] as string
+        }
 
-      return response
-    })
+        return response
+      },
+      error => {
+        if (error.response) {
+          // Append server response to error message
+          throw new Error(`AxiosError: ${error.message}. Server response: ${JSON.stringify(error.response?.data)}`)
+        } else {
+          // Something happened while setting up the request
+          throw error
+        }
+      },
+    )
   }
 
   private getV2Url(method: string) {
@@ -73,7 +84,7 @@ export class FairOSApi {
   async login(username: string, password: string): Promise<AxiosResponse> {
     const url = this.getV2Url('user/login')
 
-    return await axios.post(url, {
+    return axios.post(url, {
       userName: username,
       password,
     })
@@ -85,7 +96,7 @@ export class FairOSApi {
   async register(username: string, password: string, mnemonic: string): Promise<AxiosResponse> {
     const url = this.getV2Url('user/signup')
 
-    return await axios.post(url, {
+    return axios.post(url, {
       userName: username,
       password,
       mnemonic,