1102 lines
42 KiB
C#
1102 lines
42 KiB
C#
// Copyright Epic Games, Inc. All Rights Reserved.
|
|
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.Globalization;
|
|
using System.IO;
|
|
using System.Linq;
|
|
using System.Net;
|
|
using System.Net.Http;
|
|
using System.Net.Http.Headers;
|
|
using System.Net.Http.Json;
|
|
using System.Net.Mime;
|
|
using System.Text;
|
|
using System.Text.Json;
|
|
using System.Text.Json.Nodes;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
using Amazon.S3;
|
|
using Amazon.S3.Model;
|
|
using Azure.Storage.Blobs;
|
|
using EpicGames.AspNet;
|
|
using EpicGames.Core;
|
|
using EpicGames.Horde.Storage;
|
|
using EpicGames.Serialization;
|
|
using Jupiter.Controllers;
|
|
using Jupiter.Implementation;
|
|
using Jupiter.Tests.Functional;
|
|
using Microsoft.AspNetCore.Hosting;
|
|
using Microsoft.AspNetCore.Mvc;
|
|
using Microsoft.AspNetCore.TestHost;
|
|
using Microsoft.Extensions.Configuration;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Microsoft.Extensions.Options;
|
|
using Microsoft.VisualStudio.TestTools.UnitTesting;
|
|
using Serilog;
|
|
using Serilog.Core;
|
|
using ContentHash = Jupiter.Implementation.ContentHash;
|
|
using IBlobStore = Jupiter.Implementation.IBlobStore;
|
|
|
|
namespace Jupiter.FunctionalTests.Storage
|
|
{
|
|
[TestClass]
|
|
public class MixStorageTests : StorageTests
|
|
{
|
|
private IAmazonS3? _s3;
|
|
private readonly string _localTestDir;
|
|
|
|
protected NamespaceId TestBypassCacheNamespaceName { get; } = new NamespaceId("test-namespace-bypass-cache");
|
|
|
|
public MixStorageTests()
|
|
{
|
|
_localTestDir = Path.Combine(Path.GetTempPath(), "MixFileSystemTests", Path.GetRandomFileName());
|
|
}
|
|
protected override IEnumerable<KeyValuePair<string, string?>> GetSettings()
|
|
{
|
|
return new[]
|
|
{
|
|
new KeyValuePair<string, string?>("UnrealCloudDDC:StorageImplementations:0", UnrealCloudDDCSettings.StorageBackendImplementations.FileSystem.ToString()),
|
|
new KeyValuePair<string, string?>("UnrealCloudDDC:StorageImplementations:1", UnrealCloudDDCSettings.StorageBackendImplementations.S3.ToString()),
|
|
new KeyValuePair<string, string?>("Filesystem:RootDir", _localTestDir),
|
|
new KeyValuePair<string, string?>("S3:BucketName", $"tests-mix-{TestNamespaceName}")
|
|
};
|
|
}
|
|
|
|
protected override async Task Seed(IServiceProvider provider)
|
|
{
|
|
string s3BucketName = $"tests-mix-{TestNamespaceName}";
|
|
|
|
_s3 = provider.GetService<IAmazonS3>();
|
|
Assert.IsNotNull(_s3);
|
|
try
|
|
{
|
|
await _s3.PutBucketAsync(s3BucketName);
|
|
}
|
|
catch (AmazonS3Exception e)
|
|
{
|
|
if (e.StatusCode != HttpStatusCode.Conflict)
|
|
{
|
|
// skip 409 as that means the bucket already existed
|
|
throw;
|
|
}
|
|
}
|
|
await _s3.PutObjectAsync(new PutObjectRequest { BucketName = s3BucketName, Key = SmallFileHash.AsS3Key(), ContentBody = SmallFileContents });
|
|
await _s3.PutObjectAsync(new PutObjectRequest { BucketName = s3BucketName, Key = AnotherFileHash.AsS3Key(), ContentBody = AnotherFileContents });
|
|
await _s3.PutObjectAsync(new PutObjectRequest { BucketName = s3BucketName, Key = DeleteFileHash.AsS3Key(), ContentBody = DeletableFileContents });
|
|
await _s3.PutObjectAsync(new PutObjectRequest { BucketName = s3BucketName, Key = OldBlobFileHash.AsS3Key(), ContentBody = OldFileContents });
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task GetBlobRedirectAsync()
|
|
{
|
|
HttpResponseMessage result = await HttpClient!.GetAsync(new Uri($"api/v1/s/{TestRedirectNamespaceName}/{SmallFileHash}?supportsRedirect=true", UriKind.Relative));
|
|
Assert.AreEqual(HttpStatusCode.Redirect, result.StatusCode);
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task PutBlobBypassCacheAsync()
|
|
{
|
|
byte[] payload = Encoding.ASCII.GetBytes("Foo bar bypass cache");
|
|
using ByteArrayContent requestContent = new ByteArrayContent(payload);
|
|
requestContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
|
|
requestContent.Headers.ContentLength = payload.Length;
|
|
BlobId contentHash = BlobId.FromBlob(payload);
|
|
HttpResponseMessage putResponse = await HttpClient!.PutAsync(new Uri($"api/v1/blobs/{TestBypassCacheNamespaceName}/{contentHash}", UriKind.Relative), requestContent);
|
|
putResponse.EnsureSuccessStatusCode();
|
|
BlobUploadResponse? response = await putResponse.Content.ReadFromJsonAsync<BlobUploadResponse>();
|
|
Assert.IsNotNull(response);
|
|
Assert.AreEqual(contentHash, response.Identifier);
|
|
|
|
FileSystemStore filesystemStore = Server!.Services.GetService<FileSystemStore>()!;
|
|
AmazonS3Store s3Store = Server.Services.GetService<AmazonS3Store>()!;
|
|
|
|
// verify that it got uploaded to s3 but not to the filesystem cache as this is a bypass cache operation
|
|
Assert.IsTrue(await s3Store.ExistsAsync(TestBypassCacheNamespaceName, contentHash, forceCheck: true));
|
|
Assert.IsFalse(await filesystemStore.ExistsAsync(TestBypassCacheNamespaceName, contentHash, forceCheck: true));
|
|
|
|
HttpResponseMessage getResponse = await HttpClient.GetAsync(new Uri($"api/v1/blobs/{TestBypassCacheNamespaceName}/{contentHash}", UriKind.Relative));
|
|
getResponse.EnsureSuccessStatusCode();
|
|
CollectionAssert.AreEqual(payload, actual: await getResponse.Content.ReadAsByteArrayAsync());
|
|
|
|
// we have now done a get of the blob and it should be present in all layers
|
|
Assert.IsTrue(await s3Store.ExistsAsync(TestBypassCacheNamespaceName, contentHash, forceCheck: true));
|
|
Assert.IsTrue(await filesystemStore.ExistsAsync(TestBypassCacheNamespaceName, contentHash, forceCheck: true));
|
|
}
|
|
|
|
/// <summary>
|
|
/// write a large file (bigger then that C# can have in memory) while forcing this to not exist in the filesystem cache so that we trigger a populate
|
|
/// </summary>
|
|
|
|
[TestMethod]
|
|
[TestCategory("SlowTests")]
|
|
public async Task PutGetLargePayloadForcePopulateAsync()
|
|
{
|
|
// we submit a blob so large that it can not fit using the memory blob store
|
|
IBlobStore? blobStore = Server?.Services.GetService<IBlobStore>();
|
|
Assert.IsFalse(blobStore is MemoryBlobStore);
|
|
|
|
if (blobStore is AzureBlobStore)
|
|
{
|
|
Assert.Inconclusive("Azure blob store gets internal server errors when receiving large blobs");
|
|
}
|
|
|
|
FileSystemStore? filesystemStore = Server?.Services.GetService<FileSystemStore>();
|
|
Assert.IsNotNull(filesystemStore, "Expected to find a configured filesystem store");
|
|
|
|
FileInfo fi = new FileInfo(Path.GetTempFileName());
|
|
|
|
FileInfo tempOutputFile = new FileInfo(Path.GetTempFileName());
|
|
|
|
try
|
|
{
|
|
{
|
|
await using FileStream fs = fi.OpenWrite();
|
|
|
|
byte[] block = new byte[1024 * 1024];
|
|
Array.Fill(block, (byte)'a');
|
|
// we want a file larger then 2GB, each block is 1 MB
|
|
int countOfBlocks = 2100;
|
|
for (int i = 0; i < countOfBlocks; i++)
|
|
{
|
|
await fs.WriteAsync(block, 0, block.Length);
|
|
}
|
|
}
|
|
|
|
BlobId blobIdentifier;
|
|
{
|
|
await using FileStream fs = fi.OpenRead();
|
|
blobIdentifier = await BlobId.FromStreamAsync(fs);
|
|
}
|
|
|
|
{
|
|
await using FileStream fs = fi.OpenRead();
|
|
using StreamContent content = new StreamContent(fs);
|
|
content.Headers.ContentType = new MediaTypeHeaderValue(MediaTypeNames.Application.Octet);
|
|
HttpResponseMessage result = await HttpClient!.PutAsync(new Uri($"api/v1/blobs/{TestNamespaceName}/{blobIdentifier}", UriKind.Relative), content);
|
|
result.EnsureSuccessStatusCode();
|
|
|
|
BlobUploadResponse? response = await result.Content.ReadFromJsonAsync<BlobUploadResponse>();
|
|
Assert.IsNotNull(response);
|
|
Assert.AreEqual(blobIdentifier, response.Identifier);
|
|
}
|
|
|
|
await filesystemStore.DeleteObjectAsync(TestNamespaceName, blobIdentifier);
|
|
|
|
{
|
|
// verify we can fetch the blob again
|
|
HttpResponseMessage result = await HttpClient!.GetAsync(new Uri($"api/v1/blobs/{TestNamespaceName}/{blobIdentifier}", UriKind.Relative), HttpCompletionOption.ResponseHeadersRead);
|
|
if (result.StatusCode == HttpStatusCode.InternalServerError)
|
|
{
|
|
throw new Exception("Error: " + await result.Content.ReadAsStringAsync());
|
|
}
|
|
result.EnsureSuccessStatusCode();
|
|
Stream s = await result.Content.ReadAsStreamAsync();
|
|
|
|
{
|
|
// stream this to disk so we have something to look at in case there is an error
|
|
await using FileStream fs = tempOutputFile.OpenWrite();
|
|
await s.CopyToAsync(fs);
|
|
fs.Close();
|
|
s.Close();
|
|
|
|
s = tempOutputFile.OpenRead();
|
|
}
|
|
|
|
BlobId downloadedBlobIdentifier = await BlobId.FromStreamAsync(s);
|
|
Assert.AreEqual(blobIdentifier, downloadedBlobIdentifier);
|
|
s.Close();
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
if (fi.Exists)
|
|
{
|
|
fi.Delete();
|
|
}
|
|
|
|
if (tempOutputFile.Exists)
|
|
{
|
|
tempOutputFile.Delete();
|
|
}
|
|
}
|
|
}
|
|
|
|
protected override async Task Teardown(IServiceProvider provider)
|
|
{
|
|
if (Directory.Exists(_localTestDir))
|
|
{
|
|
Directory.Delete(_localTestDir, true);
|
|
}
|
|
|
|
await Task.CompletedTask;
|
|
}
|
|
}
|
|
|
|
[TestClass]
|
|
public class S3StorageTests : StorageTests
|
|
{
|
|
private IAmazonS3? _s3;
|
|
protected override IEnumerable<KeyValuePair<string, string?>> GetSettings()
|
|
{
|
|
return new[]
|
|
{
|
|
new KeyValuePair<string, string?>("UnrealCloudDDC:StorageImplementations:0", UnrealCloudDDCSettings.StorageBackendImplementations.S3.ToString()),
|
|
new KeyValuePair<string, string?>("S3:BucketName", $"tests-{TestNamespaceName}")
|
|
};
|
|
}
|
|
|
|
protected override async Task Seed(IServiceProvider provider)
|
|
{
|
|
string s3BucketName = $"tests-{TestNamespaceName}";
|
|
|
|
_s3 = provider.GetService<IAmazonS3>();
|
|
Assert.IsNotNull(_s3);
|
|
try
|
|
{
|
|
await _s3.PutBucketAsync(s3BucketName);
|
|
}
|
|
catch (AmazonS3Exception e)
|
|
{
|
|
if (e.StatusCode != HttpStatusCode.Conflict)
|
|
{
|
|
// skip 409 as that means the bucket already existed
|
|
throw;
|
|
}
|
|
}
|
|
await _s3.PutObjectAsync(new PutObjectRequest { BucketName = s3BucketName, Key = SmallFileHash.AsS3Key(), ContentBody = SmallFileContents });
|
|
await _s3.PutObjectAsync(new PutObjectRequest { BucketName = s3BucketName, Key = AnotherFileHash.AsS3Key(), ContentBody = AnotherFileContents });
|
|
await _s3.PutObjectAsync(new PutObjectRequest { BucketName = s3BucketName, Key = DeleteFileHash.AsS3Key(), ContentBody = DeletableFileContents });
|
|
await _s3.PutObjectAsync(new PutObjectRequest { BucketName = s3BucketName, Key = OldBlobFileHash.AsS3Key(), ContentBody = OldFileContents });
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task GetBlobRedirectAsync()
|
|
{
|
|
HttpResponseMessage result = await HttpClient!.GetAsync(new Uri($"api/v1/s/{TestRedirectNamespaceName}/{SmallFileHash}?supportsRedirect=true", UriKind.Relative));
|
|
Assert.AreEqual(HttpStatusCode.Redirect, result.StatusCode);
|
|
}
|
|
|
|
protected override async Task Teardown(IServiceProvider provider)
|
|
{
|
|
await Task.CompletedTask;
|
|
}
|
|
}
|
|
|
|
[TestClass]
|
|
public class AzureStorageTests : StorageTests
|
|
{
|
|
private AzureSettings? _settings;
|
|
private string? _connectionString;
|
|
|
|
protected override IEnumerable<KeyValuePair<string, string?>> GetSettings()
|
|
{
|
|
return new[] { new KeyValuePair<string, string?>("UnrealCloudDDC:StorageImplementations:0", UnrealCloudDDCSettings.StorageBackendImplementations.Azure.ToString()) };
|
|
}
|
|
|
|
protected override async Task Seed(IServiceProvider provider)
|
|
{
|
|
_settings = provider.GetService<IOptionsMonitor<AzureSettings>>()!.CurrentValue;
|
|
|
|
_connectionString = _settings.ConnectionString;
|
|
BlobContainerClient container = new BlobContainerClient(_connectionString, DefaultContainerName);
|
|
|
|
await container.CreateIfNotExistsAsync();
|
|
|
|
BlobClient smallBlob = container.GetBlobClient(SmallFileHash.ToString());
|
|
byte[] smallContents = Encoding.ASCII.GetBytes(SmallFileContents);
|
|
await using MemoryStream smallBlobStream = new MemoryStream(smallContents);
|
|
await smallBlob.UploadAsync(smallBlobStream, overwrite: true);
|
|
|
|
BlobClient anotherBlob = container.GetBlobClient(AnotherFileHash.ToString());
|
|
byte[] anotherContents = Encoding.ASCII.GetBytes(AnotherFileContents);
|
|
await using MemoryStream anotherContentsStream = new MemoryStream(anotherContents);
|
|
await anotherBlob.UploadAsync(anotherContentsStream, overwrite: true);
|
|
|
|
BlobClient deleteBlob = container.GetBlobClient(DeleteFileHash.ToString());
|
|
byte[] deleteContents = Encoding.ASCII.GetBytes(DeletableFileContents);
|
|
await using MemoryStream deleteContentsStream = new MemoryStream(deleteContents);
|
|
await deleteBlob.UploadAsync(deleteContentsStream, overwrite: true);
|
|
|
|
BlobClient oldBlob = container.GetBlobClient(OldBlobFileHash.ToString());
|
|
byte[] oldBlobContents = Encoding.ASCII.GetBytes(OldFileContents);
|
|
await using MemoryStream oldBlobContentsSteam = new MemoryStream(oldBlobContents);
|
|
await oldBlob.UploadAsync(oldBlobContentsSteam, overwrite: true);
|
|
}
|
|
|
|
private const string DefaultContainerName = "jupiter";
|
|
|
|
[TestMethod]
|
|
public async Task GetBlobRedirectAsync()
|
|
{
|
|
HttpResponseMessage result = await HttpClient!.GetAsync(new Uri($"api/v1/s/{TestRedirectNamespaceName}/{SmallFileHash}?supportsRedirect=true", UriKind.Relative));
|
|
Assert.AreEqual(HttpStatusCode.Redirect, result.StatusCode);
|
|
}
|
|
|
|
protected override async Task Teardown(IServiceProvider provider)
|
|
{
|
|
await Task.CompletedTask;
|
|
}
|
|
}
|
|
|
|
[TestClass]
|
|
public class FileSystemStoreTests : StorageTests
|
|
{
|
|
private readonly string _localTestDir;
|
|
private readonly NamespaceId _fooNamespace = new NamespaceId("foo");
|
|
private readonly NamespaceId _barNamespace = new NamespaceId("bar");
|
|
|
|
public FileSystemStoreTests()
|
|
{
|
|
_localTestDir = Path.Combine(Path.GetTempPath(), "IoFileSystemTests", Path.GetRandomFileName());
|
|
}
|
|
protected override IEnumerable<KeyValuePair<string, string?>> GetSettings()
|
|
{
|
|
return new[] {
|
|
new KeyValuePair<string, string?>("UnrealCloudDDC:StorageImplementations:0", UnrealCloudDDCSettings.StorageBackendImplementations.FileSystem.ToString()),
|
|
new KeyValuePair<string, string?>("Filesystem:RootDir", _localTestDir),
|
|
new KeyValuePair<string, string?>("Namespaces:Policies:bar:MaxFilesystemStorageBytes", "600")
|
|
};
|
|
}
|
|
|
|
protected override async Task Seed(IServiceProvider provider)
|
|
{
|
|
NamespaceId folderName = TestNamespaceName;
|
|
|
|
Directory.CreateDirectory(_localTestDir);
|
|
|
|
FileInfo smallFileInfo = FileSystemStore.GetFilesystemPath(_localTestDir, folderName, SmallFileHash);
|
|
smallFileInfo.Directory?.Create();
|
|
await File.WriteAllBytesAsync(
|
|
smallFileInfo.FullName,
|
|
Encoding.ASCII.GetBytes(SmallFileContents)
|
|
);
|
|
|
|
FileInfo anotherFileInfo = FileSystemStore.GetFilesystemPath(_localTestDir, folderName, AnotherFileHash);
|
|
anotherFileInfo.Directory?.Create();
|
|
await File.WriteAllBytesAsync(
|
|
anotherFileInfo.FullName,
|
|
Encoding.ASCII.GetBytes(AnotherFileContents)
|
|
);
|
|
|
|
FileInfo deleteFileInfo = FileSystemStore.GetFilesystemPath(_localTestDir, folderName, DeleteFileHash);
|
|
deleteFileInfo.Directory?.Create();
|
|
await File.WriteAllBytesAsync(
|
|
deleteFileInfo.FullName,
|
|
Encoding.ASCII.GetBytes(DeletableFileContents)
|
|
);
|
|
|
|
// a old file used to verify cutoff filtering
|
|
FileInfo oldFileInfo = FileSystemStore.GetFilesystemPath(_localTestDir, folderName, OldBlobFileHash);
|
|
oldFileInfo.Directory?.Create();
|
|
await File.WriteAllBytesAsync(
|
|
oldFileInfo.FullName,
|
|
Encoding.ASCII.GetBytes(OldFileContents)
|
|
);
|
|
File.SetLastWriteTimeUtc(oldFileInfo.FullName, DateTime.Now.AddDays(-7));
|
|
}
|
|
|
|
protected override Task Teardown(IServiceProvider provider)
|
|
{
|
|
Directory.Delete(_localTestDir, true);
|
|
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
// only a file system allows us to update the last modified time of the object to actually execute this test
|
|
[TestMethod]
|
|
public async Task ListOldBlobsAsync()
|
|
{
|
|
FileSystemStore? fsStore = Server!.Services.GetService<FileSystemStore>();
|
|
Assert.IsNotNull(fsStore);
|
|
|
|
// we fetch all objects that are more then a day old
|
|
// as the old blob is set to be a week old this should be the only object returned
|
|
DateTime cutoff = DateTime.Now.AddDays(-1);
|
|
{
|
|
BlobId[] blobs = await fsStore.ListObjectsAsync(TestNamespaceName).Where(tuple => tuple.Item2 < cutoff).Select(tuple => tuple.Item1).ToArrayAsync();
|
|
Assert.AreEqual(OldBlobFileHash, blobs[0]);
|
|
}
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task ListNamespacesAsync()
|
|
{
|
|
FileSystemStore? fsStore = Server!.Services.GetService<FileSystemStore>();
|
|
Assert.IsNotNull(fsStore);
|
|
|
|
List<NamespaceId> namespaces = await fsStore.ListNamespaces().ToListAsync();
|
|
Assert.AreEqual(1, namespaces.Count);
|
|
|
|
await fsStore.PutObjectAsync(_fooNamespace, Encoding.ASCII.GetBytes(SmallFileContents), SmallFileHash);
|
|
|
|
namespaces = await fsStore.ListNamespaces().ToListAsync();
|
|
Assert.AreEqual(2, namespaces.Count);
|
|
}
|
|
|
|
[TestMethod]
|
|
[DoNotParallelize]
|
|
public async Task GarbageCollectAsync()
|
|
{
|
|
// Remove data added from test seeding
|
|
Directory.Delete(_localTestDir, true);
|
|
Directory.CreateDirectory(_localTestDir);
|
|
|
|
FilesystemSettings fsSettings = Server!.Services.GetService<IOptionsMonitor<FilesystemSettings>>()!.CurrentValue;
|
|
fsSettings.MaxSizeBytes = 600;
|
|
FileSystemStore? fsStore = Server!.Services.GetService<FileSystemStore>();
|
|
Assert.IsNotNull(fsStore);
|
|
|
|
using CancellationTokenSource cts = new CancellationTokenSource();
|
|
Assert.IsTrue(await fsStore.CleanupInternalAsync(cts.Token) == 0); // No garbage to collect, should return false
|
|
|
|
FileInfo[] fooFiles = CreateFilesInNamespace(_fooNamespace, 10);
|
|
|
|
Assert.AreEqual(10 * 100, await fsStore.CalculateDiskSpaceUsedAsync());
|
|
|
|
Assert.IsTrue(await fsStore.CleanupInternalAsync(cts.Token) > 0);
|
|
|
|
Assert.AreEqual(5 * 100, await fsStore.CalculateDiskSpaceUsedAsync());
|
|
|
|
fooFiles.ToList().ForEach(x => x.Refresh());
|
|
Assert.IsTrue(fooFiles[0].Exists); // Most recently accessed/modified
|
|
Assert.IsTrue(fooFiles[1].Exists);
|
|
Assert.IsTrue(fooFiles[2].Exists);
|
|
// the files in the middle could have been deleted or not depending on sorting order, most recent and oldest objects are guaranteed to still be around / be deleted
|
|
Assert.IsFalse(fooFiles[8].Exists);
|
|
Assert.IsFalse(fooFiles[9].Exists); // Least recently accessed/modified
|
|
}
|
|
|
|
[TestMethod]
|
|
[DoNotParallelize]
|
|
public async Task GarbageCollectPerNamespaceAsync()
|
|
{
|
|
// Remove data added from test seeding
|
|
Directory.Delete(_localTestDir, true);
|
|
Directory.CreateDirectory(_localTestDir);
|
|
|
|
FilesystemSettings fsSettings = Server!.Services.GetService<IOptionsMonitor<FilesystemSettings>>()!.CurrentValue;
|
|
fsSettings.PerNamespaceGC = true;
|
|
fsSettings.MaxSizeBytes = 10_000; // set the default GC limit to a large number so that the foo namespace is not impacted by the GC
|
|
|
|
FileSystemStore? fsStore = Server!.Services.GetService<FileSystemStore>();
|
|
Assert.IsNotNull(fsStore);
|
|
|
|
using CancellationTokenSource cts = new CancellationTokenSource();
|
|
Assert.IsTrue(await fsStore.CleanupInternalAsync(cts.Token) == 0); // No garbage to collect, should return false
|
|
|
|
FileInfo[] fooFiles = CreateFilesInNamespace(_fooNamespace, 10);
|
|
FileInfo[] barFiles = CreateFilesInNamespace(_barNamespace, 10);
|
|
|
|
Assert.AreEqual(10 * 100, await fsStore.CalculateDiskSpaceUsedAsync(_barNamespace));
|
|
|
|
Assert.IsTrue(await fsStore.CleanupInternalAsync(cts.Token) > 0);
|
|
|
|
Assert.AreEqual(5 * 100, await fsStore.CalculateDiskSpaceUsedAsync(_barNamespace));
|
|
|
|
barFiles.ToList().ForEach(x => x.Refresh());
|
|
Assert.IsTrue(barFiles[0].Exists); // Most recently accessed/modified
|
|
Assert.IsTrue(barFiles[1].Exists);
|
|
Assert.IsTrue(barFiles[2].Exists);
|
|
// the files in the middle could have been deleted or not depending on sorting order, most recent and oldest objects are guaranteed to still be around / be deleted
|
|
Assert.IsFalse(barFiles[8].Exists);
|
|
Assert.IsFalse(barFiles[9].Exists); // Least recently accessed/modified
|
|
|
|
// verify we didn't touch the foo files
|
|
fooFiles.ToList().ForEach(x => x.Refresh());
|
|
foreach (FileInfo fooFile in fooFiles)
|
|
{
|
|
Assert.IsTrue(fooFile.Exists);
|
|
}
|
|
}
|
|
|
|
[TestMethod]
|
|
public void GetLeastRecentlyAccessedObjects()
|
|
{
|
|
FileInfo[] fooFiles = CreateFilesInNamespace(_fooNamespace, 10);
|
|
CreateFilesInNamespace(new NamespaceId("bar"), 10);
|
|
|
|
FileSystemStore? fsStore = Server!.Services.GetService<FileSystemStore>();
|
|
Assert.IsNotNull(fsStore);
|
|
|
|
Assert.AreEqual(10, (fsStore.GetLeastRecentlyAccessedObjects(_fooNamespace)).ToArray().Length);
|
|
|
|
FileInfo[] results = fsStore.GetLeastRecentlyAccessedObjects(_fooNamespace, 3).ToArray();
|
|
Assert.AreEqual(3, results.Length);
|
|
Assert.AreEqual(fooFiles[7].LastAccessTime, results[2].LastAccessTime);
|
|
Assert.AreEqual(fooFiles[8].LastAccessTime, results[1].LastAccessTime);
|
|
Assert.AreEqual(fooFiles[9].LastAccessTime, results[0].LastAccessTime);
|
|
}
|
|
|
|
[TestMethod]
|
|
public void GetObjectsOlderThen()
|
|
{
|
|
FileInfo[] fooFiles = CreateFilesInNamespace(_fooNamespace, 10);
|
|
CreateFilesInNamespace(new NamespaceId("bar"), 10);
|
|
|
|
FileSystemStore? fsStore = Server!.Services.GetService<FileSystemStore>();
|
|
Assert.IsNotNull(fsStore);
|
|
|
|
Assert.AreEqual(10, fsStore.GetObjectsOlderThen(DateTime.Now, _fooNamespace).ToArray().Length);
|
|
|
|
FileInfo[] results = fsStore.GetObjectsOlderThen(DateTime.Now.AddDays(-7), _fooNamespace).ToArray();
|
|
Assert.AreEqual(3, results.Length);
|
|
Assert.AreEqual(fooFiles[8].LastAccessTime.ToString(CultureInfo.InvariantCulture), results[0].LastAccessTime.ToString(CultureInfo.InvariantCulture));
|
|
Assert.AreEqual(fooFiles[9].LastAccessTime.ToString(CultureInfo.InvariantCulture), results[1].LastAccessTime.ToString(CultureInfo.InvariantCulture));
|
|
Assert.AreEqual(fooFiles[7].LastAccessTime.ToString(CultureInfo.InvariantCulture), results[2].LastAccessTime.ToString(CultureInfo.InvariantCulture));
|
|
}
|
|
[TestMethod]
|
|
public async Task CalculateUsedDiskSpaceAsync()
|
|
{
|
|
FileSystemStore? fsStore = Server!.Services.GetService<FileSystemStore>();
|
|
Assert.IsNotNull(fsStore);
|
|
await fsStore.PutObjectAsync(_fooNamespace, Encoding.ASCII.GetBytes(SmallFileContents), SmallFileHash);
|
|
await fsStore.PutObjectAsync(_fooNamespace, Encoding.ASCII.GetBytes(AnotherFileContents), AnotherFileHash);
|
|
|
|
Assert.AreEqual(SmallFileContents.Length + AnotherFileContents.Length, await fsStore.CalculateDiskSpaceUsedAsync(_fooNamespace));
|
|
}
|
|
|
|
private FileInfo[] CreateFilesInNamespace(NamespaceId ns, int numFiles)
|
|
{
|
|
FileInfo[] files = new FileInfo[numFiles];
|
|
for (int i = 0; i < numFiles; i++)
|
|
{
|
|
byte[] content = Encoding.ASCII.GetBytes(i + 1000 + new string('j', 96));
|
|
BlobId bi = BlobId.FromBlob(content);
|
|
FileInfo fi = FileSystemStore.GetFilesystemPath(_localTestDir, ns, bi);
|
|
fi.Directory?.Create();
|
|
File.WriteAllBytes(fi.FullName, content);
|
|
fi.LastWriteTime = DateTime.UtcNow.AddDays(-i);
|
|
fi.Refresh();
|
|
files[i] = fi;
|
|
}
|
|
|
|
return files;
|
|
}
|
|
}
|
|
|
|
public abstract class StorageTests
|
|
{
|
|
protected TestServer? Server { get; set; }
|
|
protected NamespaceId TestNamespaceName { get; } = new NamespaceId("testbucket");
|
|
protected NamespaceId TestBundleNamespaceName { get; } = new NamespaceId("test-namespace-bundle");
|
|
protected NamespaceId TestRedirectNamespaceName { get; } = new NamespaceId("test-namespace-redirect");
|
|
|
|
private HttpClient? _httpClient;
|
|
|
|
protected const string SmallFileContents = "Small file contents";
|
|
protected const string AnotherFileContents = "Another file with contents";
|
|
protected const string DeletableFileContents = "Delete Me";
|
|
protected const string OldFileContents = "a old blob used for testing cutoff filtering";
|
|
|
|
protected BlobId SmallFileHash { get; } = BlobId.FromBlob(Encoding.ASCII.GetBytes(SmallFileContents));
|
|
protected BlobId AnotherFileHash { get; } = BlobId.FromBlob(Encoding.ASCII.GetBytes(AnotherFileContents));
|
|
protected BlobId DeleteFileHash { get; } = BlobId.FromBlob(Encoding.ASCII.GetBytes(DeletableFileContents));
|
|
protected BlobId OldBlobFileHash { get; } = BlobId.FromBlob(Encoding.ASCII.GetBytes(OldFileContents));
|
|
|
|
protected HttpClient? HttpClient => _httpClient;
|
|
|
|
[TestInitialize]
|
|
public async Task SetupAsync()
|
|
{
|
|
IConfigurationRoot configuration = new ConfigurationBuilder()
|
|
// we are not reading the base appSettings here as we want exact control over what runs in the tests
|
|
.AddJsonFile("appsettings.Testing.json", true)
|
|
.AddInMemoryCollection(GetSettings())
|
|
.AddEnvironmentVariables()
|
|
.Build();
|
|
|
|
Logger logger = new LoggerConfiguration()
|
|
.ReadFrom.Configuration(configuration)
|
|
.CreateLogger();
|
|
|
|
TestServer server = new TestServer(new WebHostBuilder()
|
|
.UseConfiguration(configuration)
|
|
.UseEnvironment("Testing")
|
|
.ConfigureServices(collection => collection.AddSerilog(logger))
|
|
.UseStartup<JupiterStartup>()
|
|
);
|
|
_httpClient = server.CreateClient();
|
|
Server = server;
|
|
|
|
//foreach (NamespaceId ns in new [] { TestNamespaceName, TestListNamespaceName})
|
|
{
|
|
// Seed storage
|
|
await Seed(Server.Services);
|
|
}
|
|
}
|
|
|
|
protected abstract IEnumerable<KeyValuePair<string, string?>> GetSettings();
|
|
|
|
protected abstract Task Seed(IServiceProvider serverServices);
|
|
protected abstract Task Teardown(IServiceProvider serverServices);
|
|
|
|
[TestCleanup]
|
|
public async Task MyTeardownAsync()
|
|
{
|
|
await Teardown(Server!.Services);
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task GetSmallFileAsync()
|
|
{
|
|
HttpResponseMessage result = await _httpClient!.GetAsync(new Uri($"api/v1/s/{TestNamespaceName}/{SmallFileHash}", UriKind.Relative));
|
|
result.EnsureSuccessStatusCode();
|
|
string content = await result.Content.ReadAsStringAsync();
|
|
Assert.AreEqual(SmallFileContents, content);
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task GetNotExistentFileAsync()
|
|
{
|
|
byte[] payload = Encoding.ASCII.GetBytes("This content does not exist");
|
|
ContentHash contentHash = ContentHash.FromBlob(payload);
|
|
HttpResponseMessage result = await _httpClient!.GetAsync(new Uri($"api/v1/s/{TestNamespaceName}/{contentHash}", UriKind.Relative));
|
|
|
|
Assert.AreEqual(HttpStatusCode.NotFound, result.StatusCode);
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task GetInvalidHashAsync()
|
|
{
|
|
HttpResponseMessage result = await _httpClient!.GetAsync(new Uri($"api/v1/s/{TestNamespaceName}/smallFile", UriKind.Relative));
|
|
|
|
Assert.AreEqual(HttpStatusCode.BadRequest, result.StatusCode);
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task PutSmallBlobAsync()
|
|
{
|
|
byte[] payload = Encoding.ASCII.GetBytes("I am a small blob");
|
|
using ByteArrayContent requestContent = new ByteArrayContent(payload);
|
|
requestContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
|
|
requestContent.Headers.ContentLength = payload.Length;
|
|
BlobId contentHash = BlobId.FromBlob(payload);
|
|
HttpResponseMessage result = await _httpClient!.PutAsync(new Uri($"api/v1/s/{TestNamespaceName}/{contentHash}", UriKind.Relative), requestContent);
|
|
|
|
result.EnsureSuccessStatusCode();
|
|
BlobUploadResponse? content = await result.Content.ReadFromJsonAsync<BlobUploadResponse>();
|
|
Assert.IsNotNull(content);
|
|
Assert.AreEqual(contentHash, content.Identifier);
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task PostSmallBlobAsync()
|
|
{
|
|
byte[] payload = Encoding.ASCII.GetBytes("I am a small blob");
|
|
using ByteArrayContent requestContent = new ByteArrayContent(payload);
|
|
requestContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
|
|
requestContent.Headers.ContentLength = payload.Length;
|
|
BlobId contentHash = BlobId.FromBlob(payload);
|
|
HttpResponseMessage result = await _httpClient!.PostAsync(new Uri($"api/v1/s/{TestNamespaceName}", UriKind.Relative), requestContent);
|
|
|
|
result.EnsureSuccessStatusCode();
|
|
BlobUploadResponse? content = await result.Content.ReadFromJsonAsync<BlobUploadResponse>();
|
|
Assert.IsNotNull(content);
|
|
Assert.AreEqual(contentHash, content.Identifier);
|
|
}
|
|
|
|
|
|
[TestMethod]
|
|
public async Task PostSmallBlobCompactBinaryResponseAsync()
|
|
{
|
|
byte[] payload = Encoding.ASCII.GetBytes("I am a small blob");
|
|
using ByteArrayContent requestContent = new ByteArrayContent(payload);
|
|
requestContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
|
|
requestContent.Headers.ContentLength = payload.Length;
|
|
BlobId contentHash = BlobId.FromBlob(payload);
|
|
using HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Post, new Uri($"api/v1/s/{TestNamespaceName}", UriKind.Relative));
|
|
request.Content = requestContent;
|
|
request.Headers.Add("Accept", CustomMediaTypeNames.UnrealCompactBinary);
|
|
|
|
HttpResponseMessage result = await _httpClient!.SendAsync(request);
|
|
result.EnsureSuccessStatusCode();
|
|
BlobUploadResponse? content = await result.Content.ReadAsCompactBinaryAsync<BlobUploadResponse>();
|
|
Assert.IsNotNull(content);
|
|
Assert.AreEqual(contentHash, content.Identifier);
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task PutSmallBlobCompactBinaryResponseAsync()
|
|
{
|
|
byte[] payload = Encoding.ASCII.GetBytes("I am a small blob");
|
|
|
|
using ByteArrayContent requestContent = new ByteArrayContent(payload);
|
|
requestContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
|
|
requestContent.Headers.ContentLength = payload.Length;
|
|
BlobId contentHash = BlobId.FromBlob(payload);
|
|
using HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Put, new Uri($"api/v1/s/{TestNamespaceName}/{contentHash}", UriKind.Relative));
|
|
request.Content = requestContent;
|
|
request.Headers.Add("Accept", CustomMediaTypeNames.UnrealCompactBinary);
|
|
|
|
HttpResponseMessage result = await _httpClient!.SendAsync(request);
|
|
result.EnsureSuccessStatusCode();
|
|
BlobUploadResponse? content = await result.Content.ReadAsCompactBinaryAsync<BlobUploadResponse>();
|
|
Assert.IsNotNull(content);
|
|
Assert.AreEqual(contentHash, content.Identifier);
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task DeleteBlobAsync()
|
|
{
|
|
HttpResponseMessage result = await _httpClient!.DeleteAsync(new Uri($"api/v1/s/{TestNamespaceName}/{DeleteFileHash}", UriKind.Relative));
|
|
result.EnsureSuccessStatusCode();
|
|
|
|
Assert.AreEqual(HttpStatusCode.NoContent, result.StatusCode);
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task BlobExistsAsync()
|
|
{
|
|
{
|
|
using HttpRequestMessage message = new(HttpMethod.Head, new Uri($"api/v1/s/{TestNamespaceName}/{SmallFileHash}", UriKind.Relative));
|
|
HttpResponseMessage result = await _httpClient!.SendAsync(message);
|
|
Assert.AreEqual(HttpStatusCode.OK, result.StatusCode);
|
|
}
|
|
|
|
{
|
|
ContentHash newContent = ContentHash.FromBlob(Encoding.ASCII.GetBytes("this content has never been submitted"));
|
|
using HttpRequestMessage message = new(HttpMethod.Head, new Uri($"api/v1/s/{TestNamespaceName}/{newContent}", UriKind.Relative));
|
|
HttpResponseMessage resultNew = await _httpClient!.SendAsync(message);
|
|
Assert.AreEqual(HttpStatusCode.NotFound, resultNew.StatusCode);
|
|
string content = await resultNew.Content.ReadAsStringAsync();
|
|
ValidationProblemDetails result = JsonSerializer.Deserialize<ValidationProblemDetails>(content, JsonTestUtils.DefaultJsonSerializerSettings)!;
|
|
Assert.AreEqual($"Blob {newContent} not found", result.Title);
|
|
}
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task BlobExistsBatchAsync()
|
|
{
|
|
BlobId newContent = BlobId.FromBlob(Encoding.ASCII.GetBytes("this content has never been submitted"));
|
|
|
|
var ops = new
|
|
{
|
|
Operations = new object[]
|
|
{
|
|
new
|
|
{
|
|
Op = "HEAD",
|
|
Namespace = TestNamespaceName,
|
|
Id = SmallFileHash,
|
|
},
|
|
new
|
|
{
|
|
Op = "HEAD",
|
|
Namespace = TestNamespaceName,
|
|
Id = newContent,
|
|
}
|
|
}
|
|
};
|
|
|
|
HttpResponseMessage response = await _httpClient.PostAsJsonAsync("api/v1/s", ops);
|
|
response.EnsureSuccessStatusCode();
|
|
string content = await response.Content.ReadAsStringAsync();
|
|
JsonNode? nodes = JsonNode.Parse(content)!;
|
|
Assert.IsNotNull(nodes);
|
|
|
|
JsonArray array = nodes.AsArray();
|
|
Assert.IsNotNull(array);
|
|
Assert.AreEqual(2, array!.Count);
|
|
Assert.IsNull(array[0]);
|
|
BlobId id = new BlobId(array[1]!.AsValue().ToString()!);
|
|
Assert.AreEqual(newContent, id);
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task BatchOpAsync()
|
|
{
|
|
var ops = new
|
|
{
|
|
Operations = new object[]
|
|
{
|
|
new
|
|
{
|
|
Op = "GET",
|
|
Namespace = TestNamespaceName,
|
|
Id = SmallFileHash,
|
|
},
|
|
new
|
|
{
|
|
Op = "PUT",
|
|
Namespace = TestNamespaceName,
|
|
Id = DeleteFileHash,
|
|
Content = Encoding.ASCII.GetBytes(DeletableFileContents)
|
|
}
|
|
}
|
|
};
|
|
|
|
HttpResponseMessage response = await _httpClient.PostAsJsonAsync("api/v1/s", ops);
|
|
response.EnsureSuccessStatusCode();
|
|
string content = await response.Content.ReadAsStringAsync();
|
|
JsonNode? nodes = JsonNode.Parse(content)!;
|
|
JsonArray array = nodes.AsArray();
|
|
Assert.IsNotNull(array);
|
|
Assert.AreEqual(2, array!.Count);
|
|
string base64Content = array[0]!.AsValue().ToString()!;
|
|
string convertedContent = Encoding.ASCII.GetString(Convert.FromBase64String(base64Content));
|
|
Assert.AreEqual(SmallFileContents, convertedContent);
|
|
BlobId identifier = new BlobId(array[1]!.AsValue().ToString()!);
|
|
Assert.AreEqual(DeleteFileHash, identifier);
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task BatchOpBadRequestAsync()
|
|
{
|
|
var ops = new
|
|
{
|
|
Operations = new object[]
|
|
{
|
|
new
|
|
{
|
|
Op = "GET",
|
|
Namespace = TestNamespaceName,
|
|
Id = BlobId.FromBlob(Encoding.ASCII.GetBytes("foo"))
|
|
},
|
|
new
|
|
{
|
|
Op = "PUT",
|
|
Namespace = TestNamespaceName,
|
|
// no content
|
|
},
|
|
new
|
|
{
|
|
Op = "notAValidEnumValue"
|
|
}
|
|
}
|
|
};
|
|
|
|
HttpResponseMessage response = await _httpClient.PostAsJsonAsync("api/v1/s", ops);
|
|
Assert.AreEqual(HttpStatusCode.BadRequest, response.StatusCode);
|
|
|
|
string content = await response.Content.ReadAsStringAsync();
|
|
SerializableError? result = JsonSerializer.Deserialize<SerializableError>(content);
|
|
Assert.IsNotNull(result);
|
|
|
|
Assert.AreEqual(2, result.Keys.Count);
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task BatchOpBlobNotPresentAsync()
|
|
{
|
|
var ops = new
|
|
{
|
|
Operations = new object[]
|
|
{
|
|
new
|
|
{
|
|
Op = "HEAD",
|
|
Namespace = TestNamespaceName,
|
|
Id = "1DEE7232FE05FEFE623D2119626F103E05D3EE98", // random hash that does not exist
|
|
},
|
|
}
|
|
};
|
|
|
|
HttpResponseMessage response = await _httpClient.PostAsJsonAsync("api/v1/s", ops);
|
|
Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
|
|
|
|
string content = await response.Content.ReadAsStringAsync();
|
|
string[]? result = JsonSerializer.Deserialize<string[]>(content);
|
|
Assert.IsNotNull(result);
|
|
|
|
Assert.AreEqual(1, result.Length);
|
|
Assert.AreEqual("1DEE7232FE05FEFE623D2119626F103E05D3EE98", result[0]);
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task MultipleBlobChecksAsync()
|
|
{
|
|
BlobId newContent = BlobId.FromBlob(Encoding.ASCII.GetBytes("this content has never been submitted"));
|
|
using HttpRequestMessage message = new(HttpMethod.Post, new Uri($"api/v1/s/{TestNamespaceName}/exists?id={SmallFileHash}&id={newContent}", UriKind.Relative));
|
|
HttpResponseMessage response = await _httpClient!.SendAsync(message);
|
|
Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
|
|
Assert.AreEqual("application/json", response.Content.Headers.ContentType!.MediaType);
|
|
|
|
string s = await response.Content.ReadAsStringAsync();
|
|
HeadMultipleResponse? result = JsonSerializer.Deserialize<HeadMultipleResponse>(s, JsonTestUtils.DefaultJsonSerializerSettings);
|
|
|
|
Assert.IsNotNull(result);
|
|
Assert.AreEqual(1, result.Needs.Length);
|
|
Assert.AreEqual(newContent, result.Needs[0]);
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task MultipleBlobChecksBodyCBAsync()
|
|
{
|
|
BlobId newContent = BlobId.FromBlob(Encoding.ASCII.GetBytes("this content has never been submitted"));
|
|
using HttpRequestMessage request = new(HttpMethod.Post, new Uri($"api/v1/s/{TestNamespaceName}/exist", UriKind.Relative));
|
|
CbWriter writer = new CbWriter();
|
|
writer.BeginUniformArray(CbFieldType.Hash);
|
|
writer.WriteHashValue(SmallFileHash.AsIoHash());
|
|
writer.WriteHashValue(newContent.AsIoHash());
|
|
writer.EndUniformArray();
|
|
byte[] buf = writer.ToByteArray();
|
|
request.Content = new ByteArrayContent(buf);
|
|
request.Content.Headers.ContentType = new MediaTypeHeaderValue(CustomMediaTypeNames.UnrealCompactBinary);
|
|
|
|
HttpResponseMessage response = await _httpClient!.SendAsync(request);
|
|
Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
|
|
Assert.AreEqual("application/json", response.Content.Headers.ContentType!.MediaType);
|
|
|
|
string s = await response.Content.ReadAsStringAsync();
|
|
HeadMultipleResponse? result = JsonSerializer.Deserialize<HeadMultipleResponse>(s, JsonTestUtils.DefaultJsonSerializerSettings);
|
|
|
|
Assert.IsNotNull(result);
|
|
Assert.AreEqual(1, result.Needs.Length);
|
|
Assert.AreEqual(newContent, result.Needs[0]);
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task MultipleBlobChecksBodyJsonAsync()
|
|
{
|
|
BlobId newContent = BlobId.FromBlob(Encoding.ASCII.GetBytes("this content has never been submitted"));
|
|
using HttpRequestMessage request = new(HttpMethod.Post, new Uri($"api/v1/s/{TestNamespaceName}/exist", UriKind.Relative));
|
|
string jsonBody = JsonSerializer.Serialize(new BlobId[] { SmallFileHash, newContent });
|
|
request.Content = new StringContent(jsonBody, Encoding.UTF8, MediaTypeNames.Application.Json);
|
|
|
|
HttpResponseMessage response = await _httpClient!.SendAsync(request);
|
|
Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
|
|
Assert.AreEqual("application/json", response.Content.Headers.ContentType!.MediaType);
|
|
|
|
string s = await response.Content.ReadAsStringAsync();
|
|
HeadMultipleResponse? result = JsonSerializer.Deserialize<HeadMultipleResponse>(s, JsonTestUtils.DefaultJsonSerializerSettings);
|
|
|
|
Assert.IsNotNull(result);
|
|
Assert.AreEqual(1, result.Needs.Length);
|
|
Assert.AreEqual(newContent, result.Needs[0]);
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task MultipleBlobCompactBinaryResponseAsync()
|
|
{
|
|
BlobId newContent = BlobId.FromBlob(Encoding.ASCII.GetBytes("this content has never been submitted"));
|
|
using HttpRequestMessage request = new(HttpMethod.Post, new Uri($"api/v1/s/{TestNamespaceName}/exists?id={SmallFileHash}&id={newContent}", UriKind.Relative));
|
|
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(CustomMediaTypeNames.UnrealCompactBinary));
|
|
HttpResponseMessage response = await _httpClient!.SendAsync(request);
|
|
Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
|
|
Assert.AreEqual(CustomMediaTypeNames.UnrealCompactBinary, response.Content.Headers.ContentType!.MediaType!);
|
|
byte[] data = await response.Content.ReadAsByteArrayAsync();
|
|
CbObject cb = new CbObject(data);
|
|
|
|
Assert.AreEqual(1, cb.Count());
|
|
CbField? needs = cb["needs"];
|
|
Assert.IsNotNull(needs);
|
|
IoHash[] neededBlobs = needs!.AsArray().Select(field => field.AsHash()).ToArray();
|
|
|
|
Assert.AreEqual(1, neededBlobs.Length);
|
|
Assert.AreEqual(newContent, BlobId.FromIoHash(neededBlobs[0]));
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task FullFlowAsync()
|
|
{
|
|
byte[] payload = Encoding.ASCII.GetBytes("Foo bar");
|
|
using ByteArrayContent requestContent = new ByteArrayContent(payload);
|
|
requestContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
|
|
requestContent.Headers.ContentLength = payload.Length;
|
|
BlobId contentHash = BlobId.FromBlob(payload);
|
|
HttpResponseMessage putResponse = await _httpClient!.PutAsync(new Uri($"api/v1/s/{TestNamespaceName}/{contentHash}", UriKind.Relative), requestContent);
|
|
putResponse.EnsureSuccessStatusCode();
|
|
BlobUploadResponse? response = await putResponse.Content.ReadFromJsonAsync<BlobUploadResponse>();
|
|
Assert.IsNotNull(response);
|
|
Assert.AreEqual(contentHash, response.Identifier);
|
|
|
|
HttpResponseMessage getResponse = await _httpClient.GetAsync(new Uri($"api/v1/s/{TestNamespaceName}/{contentHash}", UriKind.Relative));
|
|
getResponse.EnsureSuccessStatusCode();
|
|
CollectionAssert.AreEqual(payload, actual: await getResponse.Content.ReadAsByteArrayAsync());
|
|
|
|
HttpResponseMessage deleteResponse = await _httpClient.DeleteAsync(new Uri($"api/v1/s/{TestNamespaceName}/{contentHash}", UriKind.Relative));
|
|
deleteResponse.EnsureSuccessStatusCode();
|
|
Assert.AreEqual(HttpStatusCode.NoContent, deleteResponse.StatusCode);
|
|
}
|
|
|
|
/// <summary>
|
|
/// write a large file (bigger then that C# can have in memory)
|
|
/// </summary>
|
|
|
|
[TestMethod]
|
|
[TestCategory("SlowTests")]
|
|
public async Task PutGetLargePayloadAsync()
|
|
{
|
|
// we submit a blob so large that it can not fit using the memory blob store
|
|
IBlobStore? blobStore = Server?.Services.GetService<IBlobStore>();
|
|
Assert.IsFalse(blobStore is MemoryBlobStore);
|
|
|
|
if (blobStore is AzureBlobStore)
|
|
{
|
|
Assert.Inconclusive("Azure blob store gets internal server errors when receiving large blobs");
|
|
}
|
|
|
|
FileInfo fi = new FileInfo(Path.GetTempFileName());
|
|
|
|
FileInfo tempOutputFile = new FileInfo(Path.GetTempFileName());
|
|
|
|
try
|
|
{
|
|
{
|
|
await using FileStream fs = fi.OpenWrite();
|
|
|
|
byte[] block = new byte[1024 * 1024];
|
|
Array.Fill(block, (byte)'a');
|
|
// we want a file larger then 2GB, each block is 1 MB
|
|
int countOfBlocks = 2100;
|
|
for (int i = 0; i < countOfBlocks; i++)
|
|
{
|
|
await fs.WriteAsync(block, 0, block.Length);
|
|
}
|
|
}
|
|
|
|
BlobId blobIdentifier;
|
|
{
|
|
await using FileStream fs = fi.OpenRead();
|
|
blobIdentifier = await BlobId.FromStreamAsync(fs);
|
|
}
|
|
|
|
{
|
|
await using FileStream fs = fi.OpenRead();
|
|
using StreamContent content = new StreamContent(fs);
|
|
content.Headers.ContentType = new MediaTypeHeaderValue(MediaTypeNames.Application.Octet);
|
|
HttpResponseMessage result = await _httpClient!.PutAsync(new Uri($"api/v1/blobs/{TestNamespaceName}/{blobIdentifier}", UriKind.Relative), content);
|
|
result.EnsureSuccessStatusCode();
|
|
|
|
BlobUploadResponse? response = await result.Content.ReadFromJsonAsync<BlobUploadResponse>();
|
|
Assert.IsNotNull(response);
|
|
Assert.AreEqual(blobIdentifier, response.Identifier);
|
|
}
|
|
|
|
{
|
|
// verify we can fetch the blob again
|
|
HttpResponseMessage result = await _httpClient!.GetAsync(new Uri($"api/v1/blobs/{TestNamespaceName}/{blobIdentifier}", UriKind.Relative), HttpCompletionOption.ResponseHeadersRead);
|
|
result.EnsureSuccessStatusCode();
|
|
Stream s = await result.Content.ReadAsStreamAsync();
|
|
|
|
{
|
|
// stream this to disk so we have something to look at in case there is an error
|
|
await using FileStream fs = tempOutputFile.OpenWrite();
|
|
await s.CopyToAsync(fs);
|
|
fs.Close();
|
|
s.Close();
|
|
|
|
s = tempOutputFile.OpenRead();
|
|
}
|
|
|
|
BlobId downloadedBlobIdentifier = await BlobId.FromStreamAsync(s);
|
|
Assert.AreEqual(blobIdentifier, downloadedBlobIdentifier);
|
|
s.Close();
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
if (fi.Exists)
|
|
{
|
|
fi.Delete();
|
|
}
|
|
|
|
if (tempOutputFile.Exists)
|
|
{
|
|
tempOutputFile.Delete();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|