Publish large models (connectors)

using Microsoft.Extensions.DependencyInjection;
using Speckle.Sdk.Api.GraphQL.Inputs;
using Speckle.Sdk.Models.Collections;
using Speckle.Sdk.Pipelines.Progress;
using Speckle.Sdk.Pipelines.Send;

// 1. Create the ingestion job
var sourceData = new SourceDataInput("my-connector", "1.0.0", fileName: null, fileSizeBytes: null);
var ingestion = await client.Ingestion.Create(
    new ModelIngestionCreateInput(model.id, project.id, "Publishing model", sourceData)
);

// 2. Create the pipeline and stream objects into it one at a time as you build the graph
using var pipeline = provider.GetRequiredService<ISendPipelineFactory>()
    .CreateInstance(ingestion, client.Account, new Progress<StreamProgressArgs>(), cancellationToken: default);

var root = new Collection { name = "My model" };

try
{
    foreach (var element in myLargeModelElements)
    {
        Base converted = ConvertToSpeckle(element);
        // Process() returns this object's own reference — embed it in the parent
        // instead of holding the fully-converted object graph in memory.
        ObjectReference reference = await pipeline.Process(converted);
        root.elements.Add(reference);
    }

    // The root is processed last, referencing all the children already on disk
    await pipeline.Process(root);
    await pipeline.WaitForUpload();

    // WaitForUpload() itself tells the server the pack is ready to process — the
    // server creates the version asynchronously once processing succeeds. Subscribe
    // to get notified when it's ready (see "Finding Out When the Version Is Ready" below).
}
catch (Exception ex)
{
    await client.Ingestion.FailWithError(ModelIngestionFailedInput.FromException(ingestion.id, project.id, ex));
    throw;
}