Calculate s3 storage

app/controllers/AnnotationIOController.scala

@@ -337,8 +337,7 @@ class AnnotationIOController @Inject()(
       else volumeTracing.boundingBox
 
     for {
-      tracingCanHaveSegmentIndex <- canHaveSegmentIndex(organizationId,
-                                                        dataset.name,
+      tracingCanHaveSegmentIndex <- canHaveSegmentIndex(dataset._id.toString,
                                                         fallbackLayerOpt.map(_.name),
                                                         remoteDataStoreClient)
       elementClassProto <- fallbackLayerOpt
@@ -358,13 +357,12 @@ class AnnotationIOController @Inject()(
   }
 
   private def canHaveSegmentIndex(
-      organizationId: String,
-      datasetName: String,
+      datasetId: String,
       fallbackLayerName: Option[String],
       remoteDataStoreClient: WKRemoteDataStoreClient)(implicit ec: ExecutionContext): Fox[Boolean] =
     fallbackLayerName match {
       case Some(layerName) =>
-        remoteDataStoreClient.hasSegmentIndexFile(organizationId, datasetName, layerName)
+        remoteDataStoreClient.hasSegmentIndexFile(datasetId, layerName)
       case None =>
         Fox.successful(true)
     }

app/controllers/DatasetController.scala

@@ -89,6 +89,7 @@ class DatasetController @Inject()(userService: UserService,
                                   analyticsService: AnalyticsService,
                                   mailchimpClient: MailchimpClient,
                                   wkExploreRemoteLayerService: WKExploreRemoteLayerService,
+                                  composeService: ComposeService,
                                   sil: Silhouette[WkEnv])(implicit ec: ExecutionContext, bodyParsers: PlayBodyParsers)
     extends Controller
     with MetadataAssertions {
@@ -145,10 +146,10 @@ class DatasetController @Inject()(userService: UserService,
         _ <- Fox.fromBool(dataSource.dataLayers.nonEmpty) ?~> "dataset.explore.zeroLayers"
         folderIdOpt <- Fox.runOptional(request.body.folderPath)(folderPath =>
           folderService.getOrCreateFromPathLiteral(folderPath, request.identity._organization)) ?~> "dataset.explore.autoAdd.getFolder.failed"
-        _ <- wkExploreRemoteLayerService.addRemoteDatasource(dataSource,
-                                                             request.body.datasetName,
-                                                             request.identity,
-                                                             folderIdOpt) ?~> "dataset.explore.autoAdd.failed"
+        _ <- wkExploreRemoteLayerService.addRemoteDatasourceToDatabase(dataSource,
+                                                                       request.body.datasetName,
+                                                                       request.identity,
+                                                                       folderIdOpt) ?~> "dataset.explore.autoAdd.failed"
       } yield Ok
     }
 
@@ -490,4 +491,11 @@ class DatasetController @Inject()(userService: UserService,
       }
     }
 
+  def compose(): Action[ComposeRequest] =
+    sil.SecuredAction.async(validateJson[ComposeRequest]) { implicit request =>
+      for {
+        (dataSource, newDatasetId) <- composeService.composeDataset(request.body, request.identity) ?~> "dataset.compose.failed"
+      } yield Ok(Json.obj("newDatasetId" -> newDatasetId))
+    }
+
 }

app/controllers/InitialDataController.scala

@@ -143,7 +143,11 @@ Samplecountry
       "This is a wonderful dummy publication, it has authors, it has a link, it has a doi number, those could go here.\nLorem [ipsum](https://github.com/scalableminds/webknossos) dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua.")
   )
   private val defaultDataStore =
-    DataStore(conf.Datastore.name, conf.Http.uri, conf.Datastore.publicUri.getOrElse(conf.Http.uri), conf.Datastore.key)
+    DataStore(conf.Datastore.name,
+              conf.Http.uri,
+              conf.Datastore.publicUri.getOrElse(conf.Http.uri),
+              conf.Datastore.key,
+              reportUsedStorageEnabled = true)
   private val defaultAiModel = AiModel(
     ObjectId("66544a56d20000af0e42ba0f"),
     defaultOrganization._id,

app/controllers/WKRemoteDataStoreController.scala

@@ -9,7 +9,7 @@ import com.scalableminds.webknossos.datastore.helpers.{LayerMagLinkInfo, MagLink
 import com.scalableminds.webknossos.datastore.models.UnfinishedUpload
 import com.scalableminds.webknossos.datastore.models.datasource.DataSourceId
 import com.scalableminds.webknossos.datastore.models.datasource.inbox.{InboxDataSourceLike => InboxDataSource}
-import com.scalableminds.webknossos.datastore.services.{DataSourcePathInfo, DataStoreStatus}
+import com.scalableminds.webknossos.datastore.services.{DataSourcePathInfo, DataSourceRegistrationInfo, DataStoreStatus}
 import com.scalableminds.webknossos.datastore.services.uploading.{
   LinkedLayerIdentifier,
   ReserveAdditionalInformation,
@@ -270,6 +270,34 @@ class WKRemoteDataStoreController @Inject()(
 
     }
 
+  // Register a datasource from the datastore as a dataset in the database.
+  // This is called when adding remote virtual datasets (that should only exist in the database)
+  // by the data store after exploration.
+  def registerDataSource(name: String,
+                         key: String,
+                         organizationId: String,
+                         directoryName: String,
+                         token: String): Action[DataSourceRegistrationInfo] =
+    Action.async(validateJson[DataSourceRegistrationInfo]) { implicit request =>
+      dataStoreService.validateAccess(name, key) { dataStore =>
+        for {
+          user <- bearerTokenService.userForToken(token)
+          organization <- organizationDAO.findOne(organizationId)(GlobalAccessContext) ?~> Messages(
+            "organization.notFound",
+            organizationId) ~> NOT_FOUND
+          _ <- Fox.fromBool(organization._id == user._organization) ?~> "notAllowed" ~> FORBIDDEN
+          dataset <- datasetService.createVirtualDataset(
+            directoryName,
+            organizationId,
+            dataStore,
+            request.body.dataSource,
+            request.body.folderId,
+            user
+          )
+        } yield Ok(dataset._id.toString)
+      }
+    }
+
   def jobExportProperties(name: String, key: String, jobId: ObjectId): Action[AnyContent] = Action.async {
     implicit request =>
       dataStoreService.validateAccess(name, key) { _ =>

app/controllers/WKRemoteTracingStoreController.scala

@@ -8,18 +8,15 @@ import com.scalableminds.webknossos.datastore.Annotation.AnnotationProto
 import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing
 import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing
 import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayer
-import com.scalableminds.webknossos.datastore.models.datasource.DataSourceId
 import com.scalableminds.webknossos.tracingstore.AnnotationUpdatesReport
 import com.scalableminds.webknossos.tracingstore.annotation.AnnotationLayerParameters
 import com.scalableminds.webknossos.tracingstore.tracings.TracingId
 import models.analytics.{AnalyticsService, UpdateAnnotationEvent, UpdateAnnotationViewOnlyEvent}
 import models.annotation.AnnotationState._
 import models.annotation._
 import models.dataset.{DatasetDAO, DatasetService}
-import models.organization.OrganizationDAO
 import models.user.UserDAO
 import models.user.time.TimeSpanService
-import play.api.i18n.Messages
 import play.api.libs.json.Json
 import play.api.mvc.{Action, AnyContent, PlayBodyParsers}
 import scalapb.GeneratedMessage
@@ -33,7 +30,6 @@ class WKRemoteTracingStoreController @Inject()(tracingStoreService: TracingStore
                                                wkSilhouetteEnvironment: WkSilhouetteEnvironment,
                                                timeSpanService: TimeSpanService,
                                                datasetService: DatasetService,
-                                               organizationDAO: OrganizationDAO,
                                                userDAO: UserDAO,
                                                annotationInformationProvider: AnnotationInformationProvider,
                                                analyticsService: AnalyticsService,
@@ -125,15 +121,14 @@ class WKRemoteTracingStoreController @Inject()(tracingStoreService: TracingStore
       }
     }
 
-  def dataSourceIdForAnnotation(name: String, key: String, annotationId: ObjectId): Action[AnyContent] =
+  def datasetIdForAnnotation(name: String, key: String, annotationId: ObjectId): Action[AnyContent] =
     Action.async { implicit request =>
       tracingStoreService.validateAccess(name, key) { _ =>
         implicit val ctx: DBAccessContext = GlobalAccessContext
         for {
           annotation <- annotationDAO.findOne(annotationId) ?~> "annotation.notFound"
-          dataset <- datasetDAO.findOne(annotation._dataset)
-          organization <- organizationDAO.findOne(dataset._organization)
-        } yield Ok(Json.toJson(DataSourceId(dataset.directoryName, organization._id)))
+          dataset <- datasetDAO.findOne(annotation._dataset) ?~> "dataset.notFound"
+        } yield Ok(dataset._id.toString)
       }
     }
 
@@ -151,20 +146,12 @@ class WKRemoteTracingStoreController @Inject()(tracingStoreService: TracingStore
       }
     }
 
-  def dataStoreUriForDataset(name: String,
-                             key: String,
-                             organizationId: Option[String],
-                             datasetDirectory: String): Action[AnyContent] =
+  def dataStoreUriForDataset(name: String, key: String, datasetId: ObjectId): Action[AnyContent] =
     Action.async { implicit request =>
       tracingStoreService.validateAccess(name, key) { _ =>
         implicit val ctx: DBAccessContext = GlobalAccessContext
         for {
-          organizationIdWithFallback <- Fox.fillOption(organizationId) {
-            datasetDAO.getOrganizationIdForDataset(datasetDirectory)(GlobalAccessContext)
-          } ?~> Messages("dataset.noAccess", datasetDirectory) ~> FORBIDDEN
-          dataset <- datasetDAO.findOneByDirectoryNameAndOrganization(datasetDirectory, organizationIdWithFallback) ?~> Messages(
-            "dataset.noAccess",
-            datasetDirectory) ~> FORBIDDEN
+          dataset <- datasetDAO.findOne(datasetId) ?~> "dataset.notFound" ~> NOT_FOUND
           dataStore <- datasetService.dataStoreFor(dataset)
         } yield Ok(Json.toJson(dataStore.url))
       }

app/models/annotation/AnnotationService.scala

@@ -121,6 +121,7 @@ class AnnotationService @Inject()(
   private def createVolumeTracing(
       dataSource: DataSource,
       datasetOrganizationId: String,
+      datasetId: ObjectId,
       datasetDataStore: DataStore,
       fallbackLayer: Option[SegmentationLayer],
       boundingBox: Option[BoundingBox] = None,
@@ -138,7 +139,7 @@ class AnnotationService @Inject()(
       remoteDatastoreClient = new WKRemoteDataStoreClient(datasetDataStore, rpc)
       fallbackLayerHasSegmentIndex <- fallbackLayer match {
         case Some(layer) =>
-          remoteDatastoreClient.hasSegmentIndexFile(datasetOrganizationId, dataSource.id.directoryName, layer.name)
+          remoteDatastoreClient.hasSegmentIndexFile(datasetId.toString, layer.name)
         case None => Fox.successful(false)
       }
       elementClassProto <- ElementClass
@@ -237,6 +238,7 @@ class AnnotationService @Inject()(
             volumeTracing <- createVolumeTracing(
               dataSource,
               dataset._organization,
+              dataset._id,
               dataStore,
               fallbackLayer,
               magRestrictions = params.magRestrictions.getOrElse(MagRestrictions.empty),
@@ -430,6 +432,7 @@ class AnnotationService @Inject()(
       volumeTracing <- createVolumeTracing(
         dataSource,
         dataset._organization,
+        datasetId,
         dataStore,
         fallbackLayer = fallbackLayer,
         boundingBox = boundingBox.flatMap { box =>

app/models/dataset/ComposeService.scala

@@ -0,0 +1,109 @@
+package models.dataset
+
+import com.scalableminds.util.accesscontext.DBAccessContext
+import com.scalableminds.util.objectid.ObjectId
+import com.scalableminds.util.tools.{Fox, FoxImplicits}
+import com.scalableminds.webknossos.datastore.dataformats.layers.{WKWDataLayer, WKWSegmentationLayer}
+import com.scalableminds.webknossos.datastore.models.VoxelSize
+import com.scalableminds.webknossos.datastore.models.datasource._
+import models.user.User
+import play.api.libs.json.{Json, OFormat}
+
+import javax.inject.Inject
+import scala.concurrent.ExecutionContext
+
+case class ComposeRequest(
+    newDatasetName: String,
+    targetFolderId: String,
+    organizationId: String,
+    voxelSize: VoxelSize,
+    layers: Seq[ComposeRequestLayer]
+)
+
+object ComposeRequest {
+  implicit val composeRequestFormat: OFormat[ComposeRequest] = Json.format[ComposeRequest]
+}
+case class ComposeRequestLayer(
+    datasetId: String,
+    sourceName: String,
+    newName: String,
+    transformations: Seq[CoordinateTransformation]
+)
+
+object ComposeRequestLayer {
+  implicit val composeLayerFormat: OFormat[ComposeRequestLayer] = Json.format[ComposeRequestLayer]
+}
+
+class ComposeService @Inject()(datasetDAO: DatasetDAO, dataStoreDAO: DataStoreDAO, datasetService: DatasetService)(
+    implicit ec: ExecutionContext)
+    extends FoxImplicits {
+
+  def composeDataset(composeRequest: ComposeRequest, user: User)(
+      implicit ctx: DBAccessContext): Fox[(DataSource, ObjectId)] =
+    for {
+      _ <- isComposable(composeRequest) ?~> "Datasets are not composable, they are not on the same data store"
+      dataSource <- createDatasource(composeRequest, composeRequest.newDatasetName, composeRequest.organizationId)
+      dataStore <- dataStoreDAO.findOneWithUploadsAllowed
+      dataset <- datasetService.createVirtualDataset(composeRequest.newDatasetName,
+                                                     composeRequest.organizationId,
+                                                     dataStore,
+                                                     dataSource,
+                                                     Some(composeRequest.targetFolderId),
+                                                     user)
+
+    } yield (dataSource, dataset._id)
+
+  private def getLayerFromComposeLayer(composeLayer: ComposeRequestLayer)(
+      implicit ctx: DBAccessContext): Fox[DataLayer] =
+    for {
+      datasetIdValidated <- ObjectId.fromString(composeLayer.datasetId) ?~> "Invalid dataset ID"
+      dataset <- datasetDAO.findOne(datasetIdValidated) ?~> "Dataset not found"
+      ds <- datasetService.fullDataSourceFor(dataset)
+      ds <- ds.toUsable.toFox ?~> "Dataset not usable"
+      layer <- ds.dataLayers.find(_.name == composeLayer.sourceName).toFox
+      applyCoordinateTransformations = (cOpt: Option[List[CoordinateTransformation]]) =>
+        cOpt match {
+          case Some(c) => Some(c ++ composeLayer.transformations.toList)
+          case None    => Some(composeLayer.transformations.toList)
+      }
+      editedLayer: DataLayer = layer match {
+        case l: DataLayerWithMagLocators =>
+          l.mapped(name = composeLayer.newName,
+                   coordinateTransformations = applyCoordinateTransformations(l.coordinateTransformations))
+        case l: WKWDataLayer =>
+          l.copy(name = composeLayer.newName,
+                 coordinateTransformations = applyCoordinateTransformations(l.coordinateTransformations))
+        case l: WKWSegmentationLayer =>
+          l.copy(name = composeLayer.newName,
+                 coordinateTransformations = applyCoordinateTransformations(l.coordinateTransformations))
+      }
+    } yield editedLayer
+
+  private def isComposable(composeRequest: ComposeRequest)(implicit ctx: DBAccessContext): Fox[Boolean] =
+    // Check that all datasets are on the same data store
+    // Using virtual datasets, we should also be able to compose datasets using non-file paths from different data
+    // stores, however, the data store is only stored for each data set and not per mag.
+    for {
+      _ <- Fox.successful(())
+      datasetIds = composeRequest.layers.map(_.datasetId).distinct
+      datasetIdsValidated <- Fox.serialCombined(datasetIds.toList)(ObjectId.fromString(_)) ?~> "Invalid dataset ID"
+      datasets <- Fox.serialCombined(datasetIdsValidated)(datasetDAO.findOne(_))
+      dataStores = datasets.map(_._dataStore)
+    } yield {
+      dataStores.distinct.size == 1
+    }
+
+  private def createDatasource(composeRequest: ComposeRequest, datasetDirectoryName: String, organizationId: String)(
+      implicit ctx: DBAccessContext): Fox[DataSource] =
+    for {
+      layers <- Fox.serialCombined(composeRequest.layers.toList)(getLayerFromComposeLayer(_))
+      dataSource = GenericDataSource(
+        DataSourceId(datasetDirectoryName, organizationId),
+        layers,
+        composeRequest.voxelSize,
+        None
+      )
+
+    } yield dataSource
+
+}