diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 754d84b62..d7cc9001b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - test: ["inmem", "mssql", "mssql-v3", "mysql", "postgres", "sqlite", "hal", "http"] + test: ["inmem", "mssql", "mssql-v3", "mysql", "postgres", "postgres-v2", "sqlite", "hal", "http"] runs-on: ubuntu-latest name: Test ${{ matrix.test }} @@ -46,7 +46,7 @@ jobs: fetch-depth: 0 - name: Pull Containers shell: bash - run: docker pull damianh/dotnet-core-lts-sdks:1 + run: docker pull alpinelinux/docker-cli:latest-x86_64 - name: Build and Test shell: bash run: ./build.sh test-${{ matrix.test }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index bea46c839..56ac107a2 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -8,7 +8,7 @@ jobs: strategy: fail-fast: false matrix: - test: ["inmem", "mssql", "mssql-v3", "mysql", "postgres", "sqlite", "hal", "http"] + test: ["inmem", "mssql", "mssql-v3", "mysql", "postgres", "postgres-v2", "sqlite", "hal", "http"] runs-on: ubuntu-latest name: Test ${{ matrix.test }} @@ -41,7 +41,7 @@ jobs: fetch-depth: 0 - name: Pull Containers shell: bash - run: docker pull damianh/dotnet-core-lts-sdks:1 + run: docker pull alpinelinux/docker-cli:latest-x86_64 - name: Build and Test shell: bash run: ./build.sh test-${{ matrix.test }} diff --git a/.vscode/launch.json b/.vscode/launch.json index 22bd38eb9..9c9a807d9 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -10,7 +10,7 @@ "request": "launch", "preLaunchTask": "build", // If you have changed target frameworks, make sure to update the program path. - "program": "${workspaceFolder}/src/LoadTests/bin/Debug/netcoreapp2.0/LoadTests.dll", + "program": "${workspaceFolder}/src/LoadTests/bin/Debug/net6.0/LoadTests.dll", "args": [], "cwd": "${workspaceFolder}/src/LoadTests", // For more information about the 'console' field, see https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md#console-terminal-window diff --git a/Dockerfile b/Dockerfile index 7cd4334a7..1354b9bd4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,19 @@ -FROM damianh/dotnet-core-lts-sdks:2 +FROM mcr.microsoft.com/dotnet/sdk:6.0-focal + +RUN apt-get update \ + && apt-get install git \ + && apt-get install \ + ca-certificates \ + curl \ + gnupg \ + lsb-release -y \ + && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg \ + && echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null \ + && apt-get update \ + && apt-get install docker-ce-cli -y \ + && apt-get install zip -y WORKDIR /repo diff --git a/SqlStreamStore.sln b/SqlStreamStore.sln index 60ca1bd2f..05d74e3a9 100644 --- a/SqlStreamStore.sln +++ b/SqlStreamStore.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.29926.136 +# Visual Studio Version 17 +VisualStudioVersion = 17.1.32120.378 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SqlStreamStore", "src\SqlStreamStore\SqlStreamStore.csproj", "{8F86833A-6686-409C-B0E4-6FBFB70F7DE1}" EndProject @@ -148,6 +148,14 @@ Global {5273D70A-96D7-4775-B67C-BDE48D20CA36}.Debug|Any CPU.Build.0 = Debug|Any CPU {5273D70A-96D7-4775-B67C-BDE48D20CA36}.Release|Any CPU.ActiveCfg = Release|Any CPU {5273D70A-96D7-4775-B67C-BDE48D20CA36}.Release|Any CPU.Build.0 = Release|Any CPU + {A8D87EDC-E911-42F3-891F-82DEBBD4E0A3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A8D87EDC-E911-42F3-891F-82DEBBD4E0A3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A8D87EDC-E911-42F3-891F-82DEBBD4E0A3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A8D87EDC-E911-42F3-891F-82DEBBD4E0A3}.Release|Any CPU.Build.0 = Release|Any CPU + {D1FA4E2A-9113-4F8E-987D-DDC1C1C583E2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D1FA4E2A-9113-4F8E-987D-DDC1C1C583E2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D1FA4E2A-9113-4F8E-987D-DDC1C1C583E2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D1FA4E2A-9113-4F8E-987D-DDC1C1C583E2}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/build-local.cmd b/build-local.cmd index ac72a1451..6eff8ed53 100644 --- a/build-local.cmd +++ b/build-local.cmd @@ -1,3 +1,3 @@ @ECHO OFF -dotnet run -p build/build.csproj -- %* \ No newline at end of file +dotnet run --project build/build.csproj -- %* \ No newline at end of file diff --git a/build.sh b/build.sh index 9b2d1f61e..c6fb9c9a9 100755 --- a/build.sh +++ b/build.sh @@ -8,4 +8,4 @@ docker run --rm --name sss-build \ --network host \ -e FEEDZ_SSS_API_KEY=$FEEDZ_SSS_API_KEY \ sss-build \ - dotnet run -p /repo/build/build.csproj -- "$@" \ No newline at end of file + dotnet run --project /repo/build/build.csproj -- "$@" \ No newline at end of file diff --git a/build/Program.cs b/build/Program.cs index e87410066..5edd76a46 100644 --- a/build/Program.cs +++ b/build/Program.cs @@ -8,6 +8,8 @@ namespace build { + using System.Text; + class Program { private const string ArtifactsDir = "artifacts"; @@ -20,6 +22,7 @@ class Program private const string TestMsSql = "test-mssql"; private const string TestMsSqlV3 = "test-mssql-v3"; private const string TestPostgres = "test-postgres"; + private const string TestPostgresV2 = "test-postgres-v2"; private const string TestSqlite = "test-sqlite"; private const string TestHal = "test-hal"; private const string TestHttp = "test-http"; @@ -59,11 +62,10 @@ void RunTest(string project) { try { - Run("dotnet", - $"test tests/{project}/{project}.csproj --configuration=Release --no-build --no-restore --verbosity=normal" - + $" --logger \"trx;logfilename=..\\..\\..\\{ArtifactsDir}\\{project}.trx\""); + Run("dotnet", $"test tests/{project}/{project}.csproj --configuration=Release --no-build --no-restore --verbosity=normal" + + $" --logger \"trx;logfilename=..\\..\\..\\{ArtifactsDir}\\{project}.trx\""); } - catch (NonZeroExitCodeException) when (ShouldCatch()) + catch (ExitCodeException) when (ShouldCatch()) { TestProjectsWithFailures.Add(project); } @@ -106,6 +108,11 @@ void RunTest(string project) DependsOn(Build), () => RunTest("SqlStreamStore.Postgres.Tests")); + Target( + TestPostgresV2, + DependsOn(Build), + () => RunTest("SqlStreamStore.Postgres.V2.Tests")); + Target( TestSqlite, DependsOn(Build), @@ -123,6 +130,7 @@ void RunTest(string project) "SqlStreamStore.MsSql", "SqlStreamStore.MySql", "SqlStreamStore.Postgres", + "SqlStreamStore.PostgresV2", "SqlStreamStore.HAL", "SqlStreamStore.Http", "SqlStreamStore.Sqlite", @@ -146,7 +154,7 @@ void RunTest(string project) foreach (var packageToPush in packagesToPush) { - Run("dotnet", $"nuget push {packageToPush} -s https://f.feedz.io/logicality/streamstore-ci/nuget/index.json -k {apiKey} --skip-duplicate", noEcho: true); + Run("dotnet", $"nuget push {packageToPush} -s https://f.feedz.io/gritti/sqlstreamstore/nuget/index.json -k {apiKey} --skip-duplicate", noEcho: true); } }); diff --git a/build/build.csproj b/build/build.csproj index 911b97451..b6b91514b 100644 --- a/build/build.csproj +++ b/build/build.csproj @@ -2,12 +2,12 @@ Exe - netcoreapp3.1 + net6.0 - - + + diff --git a/src/Directory.Build.props b/src/Directory.Build.props index c3a6e9c51..06a45c8fb 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -7,16 +7,16 @@ true cqrs;event-sourcing;event-store;stream-store $(NoWarn);1701;1702;1705;1591 - 7.3 + 9.0 v - 1.2 + 1.3 full true $(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb - - + + diff --git a/src/LoadTests/AppendsReadsDeadlocks.cs b/src/LoadTests/AppendsReadsDeadlocks.cs index e129202ad..8f6eeb09f 100644 --- a/src/LoadTests/AppendsReadsDeadlocks.cs +++ b/src/LoadTests/AppendsReadsDeadlocks.cs @@ -55,7 +55,7 @@ await dictStreamMessage.ForEachAsync(chunkSize, } } - public async Task GetManyPagesAsync(IStreamStore store, int chunksCount, int batchSize) + public async Task GetManyPagesAsync(IStreamStore store, int chunksCount, int batchSize) { long start = 0; //var events = new List(); diff --git a/src/LoadTests/LoadTest.cs b/src/LoadTests/LoadTest.cs index 3b47abd43..17bbac63f 100644 --- a/src/LoadTests/LoadTest.cs +++ b/src/LoadTests/LoadTest.cs @@ -5,50 +5,20 @@ using System.Threading.Tasks; using EasyConsole; using SqlStreamStore; + using SqlStreamStore.Streams; + public abstract class LoadTest { public abstract Task Run(CancellationToken cancellationToken); - protected async Task<(IStreamStore, Action)> GetStore(CancellationToken cancellationToken) + protected async Task<(IStreamStore, Action)> GetStore(CancellationToken cancellationToken) { - IStreamStore streamStore = null; + IStreamStore streamStore = null; IDisposable disposable = null; Output.WriteLine(ConsoleColor.Yellow, "Store type:"); await new Menu() - .AddSync("InMem", () => streamStore = new InMemoryStreamStore()) - .Add("MS SQL V2 (Docker)", - async _ => - { - var fixture = new MsSqlStreamStoreDb("dbo"); - Console.WriteLine(fixture.ConnectionString); - streamStore = await fixture.GetStreamStore(); - disposable = fixture; - }) - .Add("MS SQL V3 (Docker)", - async _ => - { - var fixture = new MsSqlStreamStoreDbV3("dbo"); - Console.WriteLine(fixture.ConnectionString); - streamStore = await fixture.GetStreamStore(); - disposable = fixture; - }) - .AddSync("MS SQL V3 (LocalDB)", - () => - { - var sqlLocalDb = new SqlLocalDb(); - Console.WriteLine(sqlLocalDb.ConnectionString); - streamStore = sqlLocalDb.StreamStore; - disposable = sqlLocalDb; - }) - .Add("MYSQL V3 (Docker)", - async ct => - { - var mysqlDb = new MySqlStreamStoreDb(); - streamStore = await mysqlDb.GetMySqlStreamStore(); - disposable = streamStore; - }) .Add("Postgres (Docker)", async ct => { @@ -67,13 +37,6 @@ public abstract class LoadTest streamStore = await postgresStreamStoreDb.GetPostgresStreamStore(true); disposable = postgresStreamStoreDb; }) - .Add("MySql (Docker)", - async ct => - { - var db = new MySqlStreamStoreDb(); - streamStore = await db.GetMySqlStreamStore(); - disposable = db; - }) /*.Add("MySql (Server)", async ct => { diff --git a/src/LoadTests/LoadTests.csproj b/src/LoadTests/LoadTests.csproj index b6a85c474..b7ca31a30 100644 --- a/src/LoadTests/LoadTests.csproj +++ b/src/LoadTests/LoadTests.csproj @@ -1,6 +1,6 @@ - + - netcoreapp3.1 + net6.0 win10-x64;osx.10.11-x64;ubuntu.16.10-x64;ubuntu.14.04-x64 exe false @@ -8,15 +8,17 @@ - + + + + + - - diff --git a/src/LoadTests/MsSqlStreamStoreDb.cs b/src/LoadTests/MsSqlStreamStoreDb.cs deleted file mode 100644 index 50f54669a..000000000 --- a/src/LoadTests/MsSqlStreamStoreDb.cs +++ /dev/null @@ -1,86 +0,0 @@ -namespace LoadTests -{ - using System; - using System.Threading.Tasks; - using Microsoft.Data.SqlClient; - using SqlStreamStore; - using SqlStreamStore.TestUtils.MsSql; - - public class MsSqlStreamStoreDb : IDisposable - { - public readonly string ConnectionString; - private readonly string _schema; - private readonly bool _deleteDatabaseOnDispose; - private readonly string _databaseName; - private readonly SqlServerContainer _containerInstance; - - public MsSqlStreamStoreDb(string schema, bool deleteDatabaseOnDispose = true) - { - _schema = schema; - _deleteDatabaseOnDispose = deleteDatabaseOnDispose; - _databaseName = $"sss-v2-{Guid.NewGuid():n}"; - _containerInstance = new SqlServerContainer(_databaseName); - - ConnectionString = CreateConnectionString(); - } - - public async Task GetStreamStore() - { - await CreateDatabase(); - - return await GetStreamStore(_schema); - } - - public async Task GetStreamStore(string schema) - { - var settings = new MsSqlStreamStoreSettings(ConnectionString) - { - Schema = schema, - }; - var store = new MsSqlStreamStore(settings); - await store.CreateSchema(); - - return store; - } - - public void Dispose() - { - if (!_deleteDatabaseOnDispose) - { - return; - } - using (var sqlConnection = new SqlConnection(ConnectionString)) - { - // Fixes: "Cannot drop database because it is currently in use" - SqlConnection.ClearPool(sqlConnection); - } - using (var connection = _containerInstance.CreateConnection()) - { - connection.Open(); - using (var command = new SqlCommand($"ALTER DATABASE [{_databaseName}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE", connection)) - { - command.ExecuteNonQuery(); - } - using (var command = new SqlCommand($"DROP DATABASE [{_databaseName}]", connection)) - { - command.ExecuteNonQuery(); - } - } - } - - private async Task CreateDatabase() - { - await _containerInstance.Start(); - await _containerInstance.CreateDatabase(); - } - - private string CreateConnectionString() - { - var connectionStringBuilder = _containerInstance.CreateConnectionStringBuilder(); - connectionStringBuilder.MultipleActiveResultSets = true; - connectionStringBuilder.InitialCatalog = _databaseName; - - return connectionStringBuilder.ToString(); - } - } -} \ No newline at end of file diff --git a/src/LoadTests/MsSqlStreamStoreDbV3.cs b/src/LoadTests/MsSqlStreamStoreDbV3.cs deleted file mode 100644 index 0c951e251..000000000 --- a/src/LoadTests/MsSqlStreamStoreDbV3.cs +++ /dev/null @@ -1,96 +0,0 @@ -namespace LoadTests -{ - using System; - using System.Threading.Tasks; - using Microsoft.Data.SqlClient; - using SqlStreamStore; - using SqlStreamStore.TestUtils.MsSql; - - public class MsSqlStreamStoreDbV3 : IDisposable - { - public readonly string ConnectionString; - private readonly string _schema; - private readonly bool _disableDeletionTracking; - private readonly string _databaseNameOverride; - private readonly bool _deleteDatabaseOnDispose; - private readonly string _databaseName; - private readonly SqlServerContainer _containerInstance; - - public MsSqlStreamStoreDbV3( - string schema, - bool disableDeletionTracking = false, - string databaseNameOverride = null, - bool deleteDatabaseOnDispose = true) - { - _schema = schema; - _disableDeletionTracking = disableDeletionTracking; - _databaseNameOverride = databaseNameOverride; - _deleteDatabaseOnDispose = deleteDatabaseOnDispose; - _databaseName = databaseNameOverride ?? $"sss-v3-{Guid.NewGuid():n}"; - _containerInstance = new SqlServerContainer(_databaseName); - - ConnectionString = CreateConnectionString(); - } - - public async Task GetStreamStore() - { - await CreateDatabase(); - - return await GetStreamStore(_schema); - } - - private async Task GetStreamStore(string schema) - { - var settings = new MsSqlStreamStoreV3Settings(ConnectionString) - { - Schema = schema, - DisableDeletionTracking = _disableDeletionTracking - }; - var store = new MsSqlStreamStoreV3(settings); - await store.CreateSchemaIfNotExists(); - - return store; - } - - public void Dispose() - { - if (!_deleteDatabaseOnDispose) - { - return; - } - - SqlConnection.ClearAllPools(); - - using (var connection = _containerInstance.CreateConnection()) - { - connection.Open(); - using (var command = new SqlCommand($"ALTER DATABASE [{_databaseName}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE", connection)) - { - command.ExecuteNonQuery(); - } - using (var command = new SqlCommand($"DROP DATABASE [{_databaseName}]", connection)) - { - command.ExecuteNonQuery(); - } - } - } - - private async Task CreateDatabase() - { - if (_databaseNameOverride == null) - { - await _containerInstance.Start(); - await _containerInstance.CreateDatabase(); - } - } - - private string CreateConnectionString() - { - var connectionStringBuilder = _containerInstance.CreateConnectionStringBuilder(); - connectionStringBuilder.MultipleActiveResultSets = true; - connectionStringBuilder.InitialCatalog = _databaseName; - - return connectionStringBuilder.ToString(); - } - } -} \ No newline at end of file diff --git a/src/LoadTests/MySqlStreamStoreDb.cs b/src/LoadTests/MySqlStreamStoreDb.cs deleted file mode 100644 index c5822d33a..000000000 --- a/src/LoadTests/MySqlStreamStoreDb.cs +++ /dev/null @@ -1,49 +0,0 @@ -namespace LoadTests -{ - using System; - using System.Threading.Tasks; - using SqlStreamStore; - using SqlStreamStore.TestUtils.MySql; - using Xunit.Abstractions; - - public class MySqlStreamStoreDb : IDisposable - { - private string ConnectionString => _databaseManager.ConnectionString; - private readonly MySqlContainer _databaseManager; - - public MySqlStreamStoreDb() - : this(new ConsoleTestoutputHelper()) - { - - } - public MySqlStreamStoreDb(ITestOutputHelper testOutputHelper) - { - _databaseManager = new MySqlContainer($"test_{Guid.NewGuid():n}"); - } - - public async Task GetMySqlStreamStore() - { - await CreateDatabase(); - - var settings = new MySqlStreamStoreSettings(ConnectionString); - - var mySqlStreamStore = new MySqlStreamStore(settings); - await mySqlStreamStore.CreateSchemaIfNotExists(); - return mySqlStreamStore; - } - - public Task CreateDatabase() => _databaseManager.CreateDatabase(); - - private class ConsoleTestoutputHelper : ITestOutputHelper - { - public void WriteLine(string message) => Console.Write(message); - - public void WriteLine(string format, params object[] args) => Console.WriteLine(format, args); - } - - public void Dispose() - { - - } - } -} \ No newline at end of file diff --git a/src/LoadTests/Program.cs b/src/LoadTests/Program.cs index 4d5f0ea8e..1ed271d94 100644 --- a/src/LoadTests/Program.cs +++ b/src/LoadTests/Program.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using EasyConsole; using Serilog; + using Serilog.Events; internal class Program { @@ -13,7 +14,7 @@ static async Task Main(string[] args) { Log.Logger = new LoggerConfiguration() .WriteTo - .File("LoadTests.txt") + .Console(LogEventLevel.Information) .CreateLogger(); var cts = new CancellationTokenSource(); diff --git a/src/LoadTests/ReadAllBackwards.cs b/src/LoadTests/ReadAllBackwards.cs index d6ca7144a..d064281ac 100644 --- a/src/LoadTests/ReadAllBackwards.cs +++ b/src/LoadTests/ReadAllBackwards.cs @@ -28,7 +28,7 @@ public override async Task Run(CancellationToken ct) var stopwatch = Stopwatch.StartNew(); int count = 0; var position = Position.End; - ReadAllPage page; + IReadAllPage page; do { page = await streamStore.ReadAllBackwards(position, diff --git a/src/LoadTests/ReadAllForwards.cs b/src/LoadTests/ReadAllForwards.cs index 2930b8b38..e8b28a158 100644 --- a/src/LoadTests/ReadAllForwards.cs +++ b/src/LoadTests/ReadAllForwards.cs @@ -28,7 +28,7 @@ public override async Task Run(CancellationToken ct) var stopwatch = Stopwatch.StartNew(); int count = 0; var position = Position.Start; - ReadAllPage page; + IReadAllPage page; do { page = await streamStore.ReadAllForwards(position, diff --git a/src/LoadTests/SqlLocalDb.cs b/src/LoadTests/SqlLocalDb.cs deleted file mode 100644 index 2b0d3b51c..000000000 --- a/src/LoadTests/SqlLocalDb.cs +++ /dev/null @@ -1,62 +0,0 @@ -namespace LoadTests -{ - using System; - using System.Threading.Tasks; - using MartinCostello.SqlLocalDb; - using Microsoft.Data.SqlClient; - using SqlStreamStore; - - public class SqlLocalDb : IDisposable - { - private readonly ISqlLocalDbInstanceInfo _instance; - private readonly ISqlLocalDbInstanceManager _manager; - private MsSqlStreamStoreV3 _msSqlStreamStoreV3; - private readonly SqlLocalDbApi _localDb; - private readonly string _databaseName = Guid.NewGuid().ToString(); - - public SqlLocalDb() - { - _localDb = new SqlLocalDbApi(); - _instance = _localDb.GetOrCreateInstance("SSS-LoadTests"); - _manager = _instance.Manage(); - Initialize().Wait(); - } - - private async Task Initialize() - { - if (!_instance.IsRunning) - { - _manager.Start(); - } - - using (var connection = _instance.CreateConnection()) - { - await connection.OpenAsync(); - var tempPath = Environment.GetEnvironmentVariable("Temp"); - var createDatabase = $"CREATE DATABASE [{_databaseName}] on (name='{_databaseName}', " - + $"filename='{tempPath}\\{_databaseName}.mdf')"; - using (var command = new SqlCommand(createDatabase, connection)) - { - await command.ExecuteNonQueryAsync(); - } - } - - var sqlConnectionStringBuilder = _instance.CreateConnectionStringBuilder(); - sqlConnectionStringBuilder.InitialCatalog = _databaseName; - ConnectionString = sqlConnectionStringBuilder.ToString(); - var settings = new MsSqlStreamStoreV3Settings(ConnectionString); - _msSqlStreamStoreV3 = new MsSqlStreamStoreV3(settings); - await _msSqlStreamStoreV3.CreateSchemaIfNotExists(); - } - - public IStreamStore StreamStore => _msSqlStreamStoreV3; - - public string ConnectionString { get; private set; } - - public void Dispose() - { - _msSqlStreamStoreV3.Dispose(); - _localDb.Dispose(); - } - } -} \ No newline at end of file diff --git a/src/LoadTests/TestGaps.cs b/src/LoadTests/TestGaps.cs index d686ddebd..007ee9151 100644 --- a/src/LoadTests/TestGaps.cs +++ b/src/LoadTests/TestGaps.cs @@ -28,8 +28,7 @@ public override async Task Run(CancellationToken ct) int readPageSize = Input.ReadInt("Read page size: ", 1, 10000); - string jsonData = new string('a', messageJsonDataSize * 1024); - + string jsonData = $@"{{""b"": ""{new string('a', messageJsonDataSize * 1024)}""}}"; // new string('a', messageJsonDataSize * 1024); var linkedToken = CancellationTokenSource.CreateLinkedTokenSource(ct); for(int i = 0; i < 10; i++) @@ -66,7 +65,7 @@ public override async Task Run(CancellationToken ct) } } - private static async Task WriteActualGaps(CancellationToken ct, IStreamStore streamStore) + private static async Task WriteActualGaps(CancellationToken ct, IStreamStore streamStore) { var stopwatch = Stopwatch.StartNew(); var count = 0; @@ -84,7 +83,7 @@ private static async Task WriteActualGaps(CancellationToken ct, IStreamStore str } while(!page.IsEnd) { - page = await page.ReadNext(ct); + page = await streamStore.ReadAllForwards(page.NextPosition, 73, false, ct); count += page.Messages.Length; for(int i = 0; i < page.Messages.Length; i++) { @@ -102,7 +101,7 @@ private static async Task WriteActualGaps(CancellationToken ct, IStreamStore str } - private static async Task RunRead(CancellationToken ct, IStreamStore streamStore, int readPageSize) + private static async Task RunRead(CancellationToken ct, IStreamStore streamStore, int readPageSize) { int count = 0; var stopwatch = Stopwatch.StartNew(); diff --git a/src/SqlStreamStore.HAL.DevServer/DevServerStartup.cs b/src/SqlStreamStore.HAL.DevServer/DevServerStartup.cs index 561fe3b45..4aa36ffeb 100644 --- a/src/SqlStreamStore.HAL.DevServer/DevServerStartup.cs +++ b/src/SqlStreamStore.HAL.DevServer/DevServerStartup.cs @@ -6,6 +6,7 @@ using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.DependencyInjection; + using SqlStreamStore.Streams; using MidFunc = System.Func< Microsoft.AspNetCore.Http.HttpContext, System.Func, @@ -14,11 +15,11 @@ internal class DevServerStartup : IStartup { - private readonly IStreamStore _streamStore; + private readonly IStreamStore _streamStore; private readonly SqlStreamStoreMiddlewareOptions _options; public DevServerStartup( - IStreamStore streamStore, + IStreamStore streamStore, SqlStreamStoreMiddlewareOptions options) { _streamStore = streamStore; diff --git a/src/SqlStreamStore.HAL.DevServer/SqlStreamStore.HAL.DevServer.csproj b/src/SqlStreamStore.HAL.DevServer/SqlStreamStore.HAL.DevServer.csproj index 17c615302..6c25c05b8 100644 --- a/src/SqlStreamStore.HAL.DevServer/SqlStreamStore.HAL.DevServer.csproj +++ b/src/SqlStreamStore.HAL.DevServer/SqlStreamStore.HAL.DevServer.csproj @@ -1,26 +1,28 @@ - + Exe - netcoreapp2.1 true false + net6.0 - - - - - - - - - + + + + + + + + + + + diff --git a/src/SqlStreamStore.HAL.DevServer/SqlStreamStoreFactory.cs b/src/SqlStreamStore.HAL.DevServer/SqlStreamStoreFactory.cs index 57beb5d84..d893b7cfd 100644 --- a/src/SqlStreamStore.HAL.DevServer/SqlStreamStoreFactory.cs +++ b/src/SqlStreamStore.HAL.DevServer/SqlStreamStoreFactory.cs @@ -7,9 +7,11 @@ namespace SqlStreamStore.HAL.DevServer using Microsoft.Data.SqlClient; using Npgsql; using Serilog; + using SqlStreamStore.Streams; + internal static class SqlStreamStoreFactory { - private delegate Task CreateStreamStore( + private delegate Task> CreateStreamStore( string connectionString, string schema, CancellationToken cancellationToken); @@ -26,11 +28,11 @@ private static readonly IDictionary s_factories = new Dictionary { [inmemory] = CreateInMemoryStreamStore, - [postgres] = CreatePostgresStreamStore, + //[postgres] = CreatePostgresStreamStore, [mssql] = CreateMssqlStreamStore }; - public static Task Create(CancellationToken cancellationToken = default) + public static Task> Create(CancellationToken cancellationToken = default) { var provider = Environment.GetEnvironmentVariable(SQLSTREAMSTORE_PROVIDER)?.ToLowerInvariant() ?? inmemory; @@ -48,13 +50,13 @@ public static Task Create(CancellationToken cancellationToken = de return factory(connectionString, schema, cancellationToken); } - private static Task CreateInMemoryStreamStore( + private static Task> CreateInMemoryStreamStore( string connectionString, string schema, CancellationToken cancellationToken) - => Task.FromResult(new InMemoryStreamStore()); + => Task.FromResult>(new InMemoryStreamStore()); - private static async Task CreateMssqlStreamStore( + private static async Task> CreateMssqlStreamStore( string connectionString, string schema, CancellationToken cancellationToken) @@ -94,7 +96,7 @@ CREATE DATABASE [{connectionStringBuilder.InitialCatalog}] return streamStore; } - private static async Task CreatePostgresStreamStore( + private static async Task> CreatePostgresStreamStore( string connectionString, string schema, CancellationToken cancellationToken) diff --git a/src/SqlStreamStore.HAL/AllStream/AllStreamResource.cs b/src/SqlStreamStore.HAL/AllStream/AllStreamResource.cs index 8901e41ce..a98673a2d 100644 --- a/src/SqlStreamStore.HAL/AllStream/AllStreamResource.cs +++ b/src/SqlStreamStore.HAL/AllStream/AllStreamResource.cs @@ -9,12 +9,12 @@ internal class AllStreamResource : IResource { - private readonly IStreamStore _streamStore; + private readonly IStreamStore _streamStore; private readonly bool _useCanonicalUrls; public SchemaSet Schema { get; } - public AllStreamResource(IStreamStore streamStore, bool useCanonicalUrls) + public AllStreamResource(IStreamStore streamStore, bool useCanonicalUrls) { if(streamStore == null) throw new ArgumentNullException(nameof(streamStore)); diff --git a/src/SqlStreamStore.HAL/AllStream/ReadAllStreamOperation.cs b/src/SqlStreamStore.HAL/AllStream/ReadAllStreamOperation.cs index 77d946442..d3d1e0c7e 100644 --- a/src/SqlStreamStore.HAL/AllStream/ReadAllStreamOperation.cs +++ b/src/SqlStreamStore.HAL/AllStream/ReadAllStreamOperation.cs @@ -66,7 +66,7 @@ public ReadAllStreamOperation(HttpContext context) public bool IsUriCanonical { get; } public PathString Path { get; } - public Task Invoke(IStreamStore streamStore, CancellationToken ct) + public Task Invoke(IStreamStore streamStore, CancellationToken ct) => ReadDirection == Constants.ReadDirection.Forwards ? streamStore.ReadAllForwards(_fromPositionInclusive, _maxCount, EmbedPayload, ct) : streamStore.ReadAllBackwards(_fromPositionInclusive, _maxCount, EmbedPayload, ct); diff --git a/src/SqlStreamStore.HAL/AllStreamMessage/AllStreamMessageResource.cs b/src/SqlStreamStore.HAL/AllStreamMessage/AllStreamMessageResource.cs index 5b09dfdf7..98e24fb75 100644 --- a/src/SqlStreamStore.HAL/AllStreamMessage/AllStreamMessageResource.cs +++ b/src/SqlStreamStore.HAL/AllStreamMessage/AllStreamMessageResource.cs @@ -8,11 +8,11 @@ internal class AllStreamMessageResource : IResource { - private readonly IStreamStore _streamStore; + private readonly IStreamStore _streamStore; public SchemaSet Schema { get; } - public AllStreamMessageResource(IStreamStore streamStore) + public AllStreamMessageResource(IStreamStore streamStore) { if(streamStore == null) throw new ArgumentNullException(nameof(streamStore)); diff --git a/src/SqlStreamStore.HAL/AllStreamMessage/ReadAllStreamMessageOperation.cs b/src/SqlStreamStore.HAL/AllStreamMessage/ReadAllStreamMessageOperation.cs index b22d55ad0..9262a8816 100644 --- a/src/SqlStreamStore.HAL/AllStreamMessage/ReadAllStreamMessageOperation.cs +++ b/src/SqlStreamStore.HAL/AllStreamMessage/ReadAllStreamMessageOperation.cs @@ -18,7 +18,7 @@ public ReadAllStreamMessageOperation(HttpContext context) public long Position { get; } public PathString Path { get; } - public async Task Invoke(IStreamStore streamStore, CancellationToken ct) + public async Task Invoke(IStreamStore streamStore, CancellationToken ct) { var page = await streamStore.ReadAllForwards(Position, 1, true, ct); diff --git a/src/SqlStreamStore.HAL/IStreamStoreOperation.cs b/src/SqlStreamStore.HAL/IStreamStoreOperation.cs index 85f11763f..bfb4c9298 100644 --- a/src/SqlStreamStore.HAL/IStreamStoreOperation.cs +++ b/src/SqlStreamStore.HAL/IStreamStoreOperation.cs @@ -3,10 +3,11 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Http; + using SqlStreamStore.Streams; internal interface IStreamStoreOperation { PathString Path { get; } - Task Invoke(IStreamStore streamStore, CancellationToken cancellationToken); + Task Invoke(IStreamStore streamStore, CancellationToken cancellationToken); } } \ No newline at end of file diff --git a/src/SqlStreamStore.HAL/Links.cs b/src/SqlStreamStore.HAL/Links.cs index 32ada51f0..82f6814d9 100644 --- a/src/SqlStreamStore.HAL/Links.cs +++ b/src/SqlStreamStore.HAL/Links.cs @@ -6,12 +6,9 @@ namespace SqlStreamStore.HAL using System.Net.Http; using Halcyon.HAL; using Microsoft.AspNetCore.Http; - using SqlStreamStore.HAL.Logging; internal class Links { - private static readonly ILog s_log = LogProvider.For(); - private readonly PathString _path; private readonly List<(string rel, string href, string title)> _links; private readonly string _relativePathToRoot; @@ -43,8 +40,6 @@ public Links Add(string rel, string href, string title = null) throw new ArgumentNullException(nameof(href)); _links.Add((rel, href, title)); - - s_log.Debug("Added link {link} to response for request {path}", _links[_links.Count - 1], _path); return this; } diff --git a/src/SqlStreamStore.HAL/LoggingMiddleware.cs b/src/SqlStreamStore.HAL/LoggingMiddleware.cs index dd03c8316..67684314f 100644 --- a/src/SqlStreamStore.HAL/LoggingMiddleware.cs +++ b/src/SqlStreamStore.HAL/LoggingMiddleware.cs @@ -4,17 +4,18 @@ namespace SqlStreamStore.HAL using System.Threading.Tasks; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Http; - using SqlStreamStore.HAL.Logging; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Logging; internal static class LoggingMiddleware { public static IApplicationBuilder UseMiddlewareLogging(this IApplicationBuilder builder, Type middlewareType) { - var log = LogProvider.GetLogger(middlewareType); + var log = builder.ApplicationServices.GetService().CreateLogger(middlewareType); Task MiddlewareLogging(HttpContext context, Func next) { - log.Debug("Middleware: {middleware}; Connection Id: {connectionId}", middlewareType, context.Connection.Id); + log.LogDebug("Middleware: {middleware}; Connection Id: {connectionId}", middlewareType, context.Connection.Id); return next(); } diff --git a/src/SqlStreamStore.HAL/SqlStreamStore.HAL.csproj b/src/SqlStreamStore.HAL/SqlStreamStore.HAL.csproj index c45a33b19..538ad4b33 100644 --- a/src/SqlStreamStore.HAL/SqlStreamStore.HAL.csproj +++ b/src/SqlStreamStore.HAL/SqlStreamStore.HAL.csproj @@ -1,26 +1,32 @@  - netstandard2.0 $(DefineConstants);LIBLOG_PORTABLE + net5.0;net6.0 HTTP Server for SQL Stream Store Stream Store - HAL Server true false + + + + + - - - + + %(RecursiveDir)%(Filename)%(Extension) + + diff --git a/src/SqlStreamStore.HAL/SqlStreamStoreHalMiddleware.cs b/src/SqlStreamStore.HAL/SqlStreamStoreHalMiddleware.cs index 8a5ecc4c0..48944d893 100644 --- a/src/SqlStreamStore.HAL/SqlStreamStoreHalMiddleware.cs +++ b/src/SqlStreamStore.HAL/SqlStreamStoreHalMiddleware.cs @@ -12,11 +12,11 @@ using SqlStreamStore.HAL.AllStreamMessage; using SqlStreamStore.HAL.Docs; using SqlStreamStore.HAL.Index; - using SqlStreamStore.HAL.Logging; using SqlStreamStore.HAL.StreamBrowser; using SqlStreamStore.HAL.StreamMessage; using SqlStreamStore.HAL.StreamMetadata; using SqlStreamStore.HAL.Streams; + using SqlStreamStore.Streams; using MidFunc = System.Func< Microsoft.AspNetCore.Http.HttpContext, System.Func, @@ -25,8 +25,6 @@ public static class SqlStreamStoreHalMiddleware { - private static ILog s_Log = LogProvider.GetLogger(typeof(SqlStreamStoreHalMiddleware)); - private static MidFunc Rfc1738 = (context, next) => { if(context.Request.QueryString != QueryString.Empty) @@ -52,8 +50,8 @@ public static class SqlStreamStoreHalMiddleware public static IApplicationBuilder UseSqlStreamStoreHal( this IApplicationBuilder builder, - IStreamStore streamStore, - SqlStreamStoreMiddlewareOptions options = default) + IStreamStore streamStore, + SqlStreamStoreMiddlewareOptions options = default) { if(builder == null) throw new ArgumentNullException(nameof(builder)); @@ -78,8 +76,6 @@ public static IApplicationBuilder UseSqlStreamStoreHal( streamMetadata, streamBrowser); - s_Log.Info(index.ToString); - return builder .UseExceptionHandling() .Use(Rfc1738) diff --git a/src/SqlStreamStore.HAL/StreamBrowser/ListStreamsOperation.cs b/src/SqlStreamStore.HAL/StreamBrowser/ListStreamsOperation.cs index 56447154d..97897e897 100644 --- a/src/SqlStreamStore.HAL/StreamBrowser/ListStreamsOperation.cs +++ b/src/SqlStreamStore.HAL/StreamBrowser/ListStreamsOperation.cs @@ -48,7 +48,7 @@ public ListStreamsOperation(HttpContext context) : 100; } - public Task Invoke(IStreamStore streamStore, CancellationToken cancellationToken) + public Task Invoke(IStreamStore streamStore, CancellationToken cancellationToken) => streamStore.ListStreams(Pattern, MaxCount, ContinuationToken, cancellationToken); } } \ No newline at end of file diff --git a/src/SqlStreamStore.HAL/StreamBrowser/StreamBrowserResource.cs b/src/SqlStreamStore.HAL/StreamBrowser/StreamBrowserResource.cs index bdbebdef9..7207ba369 100644 --- a/src/SqlStreamStore.HAL/StreamBrowser/StreamBrowserResource.cs +++ b/src/SqlStreamStore.HAL/StreamBrowser/StreamBrowserResource.cs @@ -5,14 +5,15 @@ namespace SqlStreamStore.HAL.StreamBrowser using System.Threading.Tasks; using Halcyon.HAL; using SqlStreamStore.HAL.StreamBrowser; + using SqlStreamStore.Streams; internal class StreamBrowserResource : IResource { - private readonly IStreamStore _streamStore; + private readonly IStreamStore _streamStore; public SchemaSet Schema { get; } - public StreamBrowserResource(IStreamStore streamStore) + public StreamBrowserResource(IStreamStore streamStore) { _streamStore = streamStore; Schema = new SchemaSet(); diff --git a/src/SqlStreamStore.HAL/StreamMessage/MessageId/DeleteStreamMessageByMessageIdOperation.cs b/src/SqlStreamStore.HAL/StreamMessage/MessageId/DeleteStreamMessageByMessageIdOperation.cs index 3d4ceda06..25696ed25 100644 --- a/src/SqlStreamStore.HAL/StreamMessage/MessageId/DeleteStreamMessageByMessageIdOperation.cs +++ b/src/SqlStreamStore.HAL/StreamMessage/MessageId/DeleteStreamMessageByMessageIdOperation.cs @@ -5,6 +5,7 @@ namespace SqlStreamStore.HAL.StreamMessage.MessageId using System.Threading.Tasks; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Routing; + using SqlStreamStore.Streams; internal class DeleteStreamMessageByMessageIdOperation : IStreamStoreOperation { @@ -20,7 +21,7 @@ public DeleteStreamMessageByMessageIdOperation(HttpContext context) public Guid MessageId { get; } public PathString Path { get; } - public async Task Invoke(IStreamStore streamStore, CancellationToken ct) + public async Task Invoke(IStreamStore streamStore, CancellationToken ct) { await streamStore.DeleteMessage(StreamId, MessageId, ct); diff --git a/src/SqlStreamStore.HAL/StreamMessage/StreamMessageResource.cs b/src/SqlStreamStore.HAL/StreamMessage/StreamMessageResource.cs index 090772d5d..edd44e801 100644 --- a/src/SqlStreamStore.HAL/StreamMessage/StreamMessageResource.cs +++ b/src/SqlStreamStore.HAL/StreamMessage/StreamMessageResource.cs @@ -10,10 +10,10 @@ namespace SqlStreamStore.HAL.StreamMessage internal class StreamMessageResource : IResource { - private readonly IStreamStore _streamStore; + private readonly IStreamStore _streamStore; public SchemaSet Schema { get; } - public StreamMessageResource(IStreamStore streamStore) + public StreamMessageResource(IStreamStore streamStore) { if(streamStore == null) throw new ArgumentNullException(nameof(streamStore)); diff --git a/src/SqlStreamStore.HAL/StreamMessage/Version/DeleteStreamMessageByVersionOperation.cs b/src/SqlStreamStore.HAL/StreamMessage/Version/DeleteStreamMessageByVersionOperation.cs index a36835a0c..e9424f756 100644 --- a/src/SqlStreamStore.HAL/StreamMessage/Version/DeleteStreamMessageByVersionOperation.cs +++ b/src/SqlStreamStore.HAL/StreamMessage/Version/DeleteStreamMessageByVersionOperation.cs @@ -5,6 +5,7 @@ namespace SqlStreamStore.HAL.StreamMessage.Version using System.Threading.Tasks; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Routing; + using SqlStreamStore.Streams; internal class DeleteStreamMessageByVersionOperation : IStreamStoreOperation { @@ -20,7 +21,7 @@ public DeleteStreamMessageByVersionOperation(HttpContext context) public int StreamVersion { get; } public PathString Path { get; } - public async Task Invoke(IStreamStore streamStore, CancellationToken ct) + public async Task Invoke(IStreamStore streamStore, CancellationToken ct) { var messageId = (await streamStore.ReadStreamBackwards( StreamId, diff --git a/src/SqlStreamStore.HAL/StreamMessage/Version/ReadStreamMessageByStreamVersionOperation.cs b/src/SqlStreamStore.HAL/StreamMessage/Version/ReadStreamMessageByStreamVersionOperation.cs index 6a809cd3e..8032e064d 100644 --- a/src/SqlStreamStore.HAL/StreamMessage/Version/ReadStreamMessageByStreamVersionOperation.cs +++ b/src/SqlStreamStore.HAL/StreamMessage/Version/ReadStreamMessageByStreamVersionOperation.cs @@ -23,7 +23,7 @@ public ReadStreamMessageByStreamVersionOperation(HttpContext context) public int StreamVersion { get; } public string StreamId { get; } - public async Task Invoke(IStreamStore streamStore, CancellationToken ct) + public async Task Invoke(IStreamStore streamStore, CancellationToken ct) => (await streamStore.ReadStreamBackwards(StreamId, StreamVersion, 1, true, ct)) .Messages.FirstOrDefault(message => StreamVersion == SqlStreamStore.Streams.StreamVersion.End || message.StreamVersion == StreamVersion); diff --git a/src/SqlStreamStore.HAL/StreamMetadata/GetStreamMetadataOperation.cs b/src/SqlStreamStore.HAL/StreamMetadata/GetStreamMetadataOperation.cs index 0b9df371d..1861550a8 100644 --- a/src/SqlStreamStore.HAL/StreamMetadata/GetStreamMetadataOperation.cs +++ b/src/SqlStreamStore.HAL/StreamMetadata/GetStreamMetadataOperation.cs @@ -17,7 +17,7 @@ public GetStreamMetadataOperation(HttpContext context) public string StreamId { get; } public PathString Path { get; } - public Task Invoke(IStreamStore streamStore, CancellationToken ct) + public Task Invoke(IStreamStore streamStore, CancellationToken ct) => streamStore.GetStreamMetadata(StreamId, ct); } } \ No newline at end of file diff --git a/src/SqlStreamStore.HAL/StreamMetadata/SetStreamMetadataOperation.cs b/src/SqlStreamStore.HAL/StreamMetadata/SetStreamMetadataOperation.cs index 83c1fde32..fa090e80a 100644 --- a/src/SqlStreamStore.HAL/StreamMetadata/SetStreamMetadataOperation.cs +++ b/src/SqlStreamStore.HAL/StreamMetadata/SetStreamMetadataOperation.cs @@ -7,6 +7,7 @@ namespace SqlStreamStore.HAL.StreamMetadata using Microsoft.AspNetCore.Routing; using Newtonsoft.Json; using Newtonsoft.Json.Linq; + using SqlStreamStore.Streams; internal class SetStreamMetadataOperation : IStreamStoreOperation { @@ -41,7 +42,7 @@ private SetStreamMetadataOperation(HttpContext context, JObject body) public int? MaxCount { get; } public int? MaxAge { get; } - public async Task Invoke(IStreamStore streamStore, CancellationToken ct) + public async Task Invoke(IStreamStore streamStore, CancellationToken ct) { await streamStore.SetStreamMetadata( StreamId, diff --git a/src/SqlStreamStore.HAL/StreamMetadata/StreamMetadataResource.cs b/src/SqlStreamStore.HAL/StreamMetadata/StreamMetadataResource.cs index 1e0af2fad..8c5148fd0 100644 --- a/src/SqlStreamStore.HAL/StreamMetadata/StreamMetadataResource.cs +++ b/src/SqlStreamStore.HAL/StreamMetadata/StreamMetadataResource.cs @@ -5,13 +5,14 @@ namespace SqlStreamStore.HAL.StreamMetadata using System.Threading.Tasks; using Halcyon.HAL; using Newtonsoft.Json.Linq; + using SqlStreamStore.Streams; internal class StreamMetadataResource : IResource { - private readonly IStreamStore _streamStore; + private readonly IStreamStore _streamStore; public SchemaSet Schema { get; } - public StreamMetadataResource(IStreamStore streamStore) + public StreamMetadataResource(IStreamStore streamStore) { if(streamStore == null) throw new ArgumentNullException(nameof(streamStore)); diff --git a/src/SqlStreamStore.HAL/Streams/AppendStreamOperation.cs b/src/SqlStreamStore.HAL/Streams/AppendStreamOperation.cs index edc98e1d6..7b6559d48 100644 --- a/src/SqlStreamStore.HAL/Streams/AppendStreamOperation.cs +++ b/src/SqlStreamStore.HAL/Streams/AppendStreamOperation.cs @@ -86,7 +86,7 @@ private static NewStreamMessageDto ParseNewStreamMessage(JToken newStreamMessage public NewStreamMessageDto[] NewStreamMessages { get; } public PathString Path { get; } - public Task Invoke(IStreamStore streamStore, CancellationToken ct) + public Task Invoke(IStreamStore streamStore, CancellationToken ct) => streamStore.AppendToStream( StreamId, ExpectedVersion, diff --git a/src/SqlStreamStore.HAL/Streams/DeleteStreamOperation.cs b/src/SqlStreamStore.HAL/Streams/DeleteStreamOperation.cs index 6d42769ac..ef07dcfb6 100644 --- a/src/SqlStreamStore.HAL/Streams/DeleteStreamOperation.cs +++ b/src/SqlStreamStore.HAL/Streams/DeleteStreamOperation.cs @@ -4,6 +4,7 @@ namespace SqlStreamStore.HAL.Streams using System.Threading.Tasks; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Routing; + using SqlStreamStore.Streams; internal class DeleteStreamOperation : IStreamStoreOperation { @@ -18,7 +19,7 @@ public DeleteStreamOperation(HttpContext context) public int ExpectedVersion { get; } public PathString Path { get; } - public async Task Invoke(IStreamStore streamStore, CancellationToken ct) + public async Task Invoke(IStreamStore streamStore, CancellationToken ct) { await streamStore.DeleteStream(StreamId, ExpectedVersion, ct); diff --git a/src/SqlStreamStore.HAL/Streams/ReadStreamOperation.cs b/src/SqlStreamStore.HAL/Streams/ReadStreamOperation.cs index ef2defca6..8f3189181 100644 --- a/src/SqlStreamStore.HAL/Streams/ReadStreamOperation.cs +++ b/src/SqlStreamStore.HAL/Streams/ReadStreamOperation.cs @@ -67,7 +67,7 @@ public ReadStreamOperation(HttpContext context) public string Self { get; } public bool IsUriCanonical { get; } - public Task Invoke(IStreamStore streamStore, CancellationToken ct) + public Task Invoke(IStreamStore streamStore, CancellationToken ct) => ReadDirection == Constants.ReadDirection.Forwards ? streamStore.ReadStreamForwards(StreamId, _fromVersionInclusive, _maxCount, EmbedPayload, ct) : streamStore.ReadStreamBackwards(StreamId, _fromVersionInclusive, _maxCount, EmbedPayload, ct); diff --git a/src/SqlStreamStore.HAL/Streams/StreamResource.cs b/src/SqlStreamStore.HAL/Streams/StreamResource.cs index 116e44bcc..3b8e6336e 100644 --- a/src/SqlStreamStore.HAL/Streams/StreamResource.cs +++ b/src/SqlStreamStore.HAL/Streams/StreamResource.cs @@ -9,12 +9,12 @@ namespace SqlStreamStore.HAL.Streams internal class StreamResource : IResource { - private readonly IStreamStore _streamStore; + private readonly IStreamStore _streamStore; private readonly string _relativePathToRoot; private readonly bool _useCanonicalUrls; public SchemaSet Schema { get; } - public StreamResource(IStreamStore streamStore, bool useCanonicalUrls) + public StreamResource(IStreamStore streamStore, bool useCanonicalUrls) { if(streamStore == null) throw new ArgumentNullException(nameof(streamStore)); diff --git a/src/SqlStreamStore.Http/HttpClientSqlStreamStore.Subscriptions.cs b/src/SqlStreamStore.Http/HttpClientSqlStreamStore.Subscriptions.cs index 8c1f179e5..5ed90b6e8 100644 --- a/src/SqlStreamStore.Http/HttpClientSqlStreamStore.Subscriptions.cs +++ b/src/SqlStreamStore.Http/HttpClientSqlStreamStore.Subscriptions.cs @@ -38,7 +38,7 @@ public IAllStreamSubscription SubscribeToAll( { GuardAgainstDisposed(); - return new AllStreamSubscription( + return new AllStreamSubscription( continueAfterPosition, this, _streamStoreNotifier.Value, diff --git a/src/SqlStreamStore.Http/HttpClientSqlStreamStore.cs b/src/SqlStreamStore.Http/HttpClientSqlStreamStore.cs index 2c91b0649..5746ef484 100644 --- a/src/SqlStreamStore.Http/HttpClientSqlStreamStore.cs +++ b/src/SqlStreamStore.Http/HttpClientSqlStreamStore.cs @@ -14,7 +14,7 @@ using SqlStreamStore.Streams; using SqlStreamStore.Subscriptions; - public sealed partial class HttpClientSqlStreamStore : IStreamStore + public sealed partial class HttpClientSqlStreamStore : IStreamStore, IReadonlyStreamStore { private static readonly JsonSerializer s_serializer = JsonSerializer.Create(new JsonSerializerSettings { diff --git a/src/SqlStreamStore.Http/SqlStreamStore.Http.csproj b/src/SqlStreamStore.Http/SqlStreamStore.Http.csproj index 6b58d086f..ab448f83d 100644 --- a/src/SqlStreamStore.Http/SqlStreamStore.Http.csproj +++ b/src/SqlStreamStore.Http/SqlStreamStore.Http.csproj @@ -2,17 +2,17 @@ Http (HAL) provider for SQL Stream Store SQL Stream Store - Http (HAL) Provider - netstandard2.0 + netstandard2.0 true SqlStreamStore.Http SqlStreamStore.Http SqlStreamStore - - - - + + + + diff --git a/src/SqlStreamStore.MsSql/MsSqlStreamStore.ReadAll.cs b/src/SqlStreamStore.MsSql/MsSqlStreamStore.ReadAll.cs index 452765771..c176f0cbd 100644 --- a/src/SqlStreamStore.MsSql/MsSqlStreamStore.ReadAll.cs +++ b/src/SqlStreamStore.MsSql/MsSqlStreamStore.ReadAll.cs @@ -15,8 +15,8 @@ protected override async Task ReadAllForwardsInternal( long fromPosition, int maxCount, bool prefetch, - ReadNextAllPage readNext, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + long fromMaxPositionInclusive = -1) { maxCount = maxCount == int.MaxValue ? maxCount - 1 : maxCount; long position = fromPosition; @@ -43,7 +43,6 @@ protected override async Task ReadAllForwardsInternal( fromPosition, true, ReadDirection.Forward, - readNext, messages.ToArray()); } @@ -103,7 +102,6 @@ protected override async Task ReadAllForwardsInternal( nextPosition, isEnd, ReadDirection.Forward, - readNext, messages.ToArray()); } } @@ -113,7 +111,6 @@ protected override async Task ReadAllBackwardsInternal( long fromPositionExclusive, int maxCount, bool prefetch, - ReadNextAllPage readNext, CancellationToken cancellationToken) { maxCount = maxCount == int.MaxValue ? maxCount - 1 : maxCount; @@ -143,7 +140,6 @@ protected override async Task ReadAllBackwardsInternal( Position.Start, true, ReadDirection.Backward, - readNext, messages.ToArray()); } @@ -199,7 +195,6 @@ protected override async Task ReadAllBackwardsInternal( nextPosition, isEnd, ReadDirection.Backward, - readNext, messages.ToArray()); } } diff --git a/src/SqlStreamStore.MsSql/MsSqlStreamStore.Subscriptions.cs b/src/SqlStreamStore.MsSql/MsSqlStreamStore.Subscriptions.cs index 93ddeadd4..d365f7baa 100644 --- a/src/SqlStreamStore.MsSql/MsSqlStreamStore.Subscriptions.cs +++ b/src/SqlStreamStore.MsSql/MsSqlStreamStore.Subscriptions.cs @@ -1,5 +1,6 @@ namespace SqlStreamStore { + using SqlStreamStore.Streams; using SqlStreamStore.Subscriptions; public sealed partial class MsSqlStreamStore @@ -33,7 +34,7 @@ protected override IAllStreamSubscription SubscribeToAllInternal( bool prefetchJsonData, string name) { - return new AllStreamSubscription( + return new AllStreamSubscription( fromPosition, this, GetStoreObservable, diff --git a/src/SqlStreamStore.MsSql/MsSqlStreamStore.cs b/src/SqlStreamStore.MsSql/MsSqlStreamStore.cs index c3a72dc26..3a77f6f86 100644 --- a/src/SqlStreamStore.MsSql/MsSqlStreamStore.cs +++ b/src/SqlStreamStore.MsSql/MsSqlStreamStore.cs @@ -17,7 +17,7 @@ /// Represents a Microsoft SQL Server stream store implementation. /// [Obsolete("Use MsSqlStreamStoreV3 instead. Note: this will require a schema and data migration.", false)] - public sealed partial class MsSqlStreamStore : StreamStoreBase + public sealed partial class MsSqlStreamStore : StreamStoreBase { private readonly Func _createConnection; private readonly Lazy _streamStoreNotifier; diff --git a/src/SqlStreamStore.MsSql/MsSqlStreamStoreV3.ReadAll.cs b/src/SqlStreamStore.MsSql/MsSqlStreamStoreV3.ReadAll.cs index a3c4d791f..926195464 100644 --- a/src/SqlStreamStore.MsSql/MsSqlStreamStoreV3.ReadAll.cs +++ b/src/SqlStreamStore.MsSql/MsSqlStreamStoreV3.ReadAll.cs @@ -15,8 +15,8 @@ protected override async Task ReadAllForwardsInternal( long fromPosition, int maxCount, bool prefetch, - ReadNextAllPage readNext, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + long fromMaxPositionInclusive = -1) { maxCount = maxCount == int.MaxValue ? maxCount - 1 : maxCount; long position = fromPosition; @@ -42,7 +42,6 @@ protected override async Task ReadAllForwardsInternal( fromPosition, true, ReadDirection.Forward, - readNext, Array.Empty()); } @@ -101,7 +100,6 @@ protected override async Task ReadAllForwardsInternal( nextPosition, isEnd, ReadDirection.Forward, - readNext, filteredMessages.ToArray()); } } @@ -111,7 +109,6 @@ protected override async Task ReadAllBackwardsInternal( long fromPositionExclusive, int maxCount, bool prefetch, - ReadNextAllPage readNext, CancellationToken cancellationToken) { maxCount = maxCount == int.MaxValue ? maxCount - 1 : maxCount; @@ -141,7 +138,6 @@ protected override async Task ReadAllBackwardsInternal( Position.Start, true, ReadDirection.Backward, - readNext, Array.Empty()); } @@ -203,7 +199,6 @@ protected override async Task ReadAllBackwardsInternal( nextPosition, isEnd, ReadDirection.Backward, - readNext, filteredMessages.ToArray()); } } diff --git a/src/SqlStreamStore.MsSql/MsSqlStreamStoreV3.Subscriptions.cs b/src/SqlStreamStore.MsSql/MsSqlStreamStoreV3.Subscriptions.cs index 9e55704d4..0732d583e 100644 --- a/src/SqlStreamStore.MsSql/MsSqlStreamStoreV3.Subscriptions.cs +++ b/src/SqlStreamStore.MsSql/MsSqlStreamStoreV3.Subscriptions.cs @@ -1,5 +1,6 @@ namespace SqlStreamStore { + using SqlStreamStore.Streams; using SqlStreamStore.Subscriptions; public sealed partial class MsSqlStreamStoreV3 @@ -33,7 +34,7 @@ protected override IAllStreamSubscription SubscribeToAllInternal( bool prefetchJsonData, string name) { - return new AllStreamSubscription( + return new AllStreamSubscription( fromPosition, this, GetStoreObservable, diff --git a/src/SqlStreamStore.MsSql/MsSqlStreamStoreV3.cs b/src/SqlStreamStore.MsSql/MsSqlStreamStoreV3.cs index eb9da9c28..aad2e6c46 100644 --- a/src/SqlStreamStore.MsSql/MsSqlStreamStoreV3.cs +++ b/src/SqlStreamStore.MsSql/MsSqlStreamStoreV3.cs @@ -18,7 +18,7 @@ /// Represents a Microsoft SQL Server stream store implementation that /// uses V3 of the store schema. /// - public sealed partial class MsSqlStreamStoreV3 : StreamStoreBase + public sealed partial class MsSqlStreamStoreV3 : StreamStoreBase { private readonly Func _createConnection; private readonly Lazy _streamStoreNotifier; diff --git a/src/SqlStreamStore.MsSql/SqlStreamStore.MsSql.csproj b/src/SqlStreamStore.MsSql/SqlStreamStore.MsSql.csproj index 3c7d47698..87f0d658c 100644 --- a/src/SqlStreamStore.MsSql/SqlStreamStore.MsSql.csproj +++ b/src/SqlStreamStore.MsSql/SqlStreamStore.MsSql.csproj @@ -2,7 +2,7 @@ SQL Server (2012+) provider for SQL Stream Store SQL Stream Store - SQL Server (2012+) Provider - netstandard2.0 + net5.0 true SqlStreamStore.MsSql SqlStreamStore.MsSql @@ -17,6 +17,6 @@ - + \ No newline at end of file diff --git a/src/SqlStreamStore.MySql/MySqlStreamStore.ReadAll.cs b/src/SqlStreamStore.MySql/MySqlStreamStore.ReadAll.cs index d50ce2234..203c1134c 100644 --- a/src/SqlStreamStore.MySql/MySqlStreamStore.ReadAll.cs +++ b/src/SqlStreamStore.MySql/MySqlStreamStore.ReadAll.cs @@ -16,8 +16,9 @@ protected override async Task ReadAllForwardsInternal( long fromPositionExclusive, int maxCount, bool prefetch, - ReadNextAllPage readNext, - CancellationToken cancellationToken) + ReadNextAllPage readNext, + CancellationToken cancellationToken, + long fromMaxPositionInclusive = -1) { maxCount = maxCount == int.MaxValue ? maxCount - 1 : maxCount; @@ -98,7 +99,7 @@ protected override async Task ReadAllBackwardsInternal( long fromPositionExclusive, int maxCount, bool prefetch, - ReadNextAllPage readNext, + ReadNextAllPage readNext, CancellationToken cancellationToken) { maxCount = maxCount == int.MaxValue ? maxCount - 1 : maxCount; diff --git a/src/SqlStreamStore.MySql/MySqlStreamStore.Subscriptions.cs b/src/SqlStreamStore.MySql/MySqlStreamStore.Subscriptions.cs index 79e4078d2..3b9b92440 100644 --- a/src/SqlStreamStore.MySql/MySqlStreamStore.Subscriptions.cs +++ b/src/SqlStreamStore.MySql/MySqlStreamStore.Subscriptions.cs @@ -2,6 +2,7 @@ namespace SqlStreamStore { using System; using SqlStreamStore.Infrastructure; + using SqlStreamStore.Streams; using SqlStreamStore.Subscriptions; partial class MySqlStreamStore @@ -31,7 +32,7 @@ protected override IAllStreamSubscription SubscribeToAllInternal( HasCaughtUp hasCaughtUp, bool prefetchJsonData, string name) - => new AllStreamSubscription( + => new AllStreamSubscription( fromPosition, this, GetStoreObservable, diff --git a/src/SqlStreamStore.MySql/MySqlStreamStore.cs b/src/SqlStreamStore.MySql/MySqlStreamStore.cs index 40f4ef496..29d123c99 100644 --- a/src/SqlStreamStore.MySql/MySqlStreamStore.cs +++ b/src/SqlStreamStore.MySql/MySqlStreamStore.cs @@ -9,6 +9,7 @@ using SqlStreamStore.Infrastructure; using SqlStreamStore.Logging; using SqlStreamStore.MySqlScripts; + using SqlStreamStore.Streams; using SqlStreamStore.Subscriptions; using StreamStoreStore.Json; @@ -16,7 +17,7 @@ /// /// Represents a MySql stream store implementation. /// - public partial class MySqlStreamStore : StreamStoreBase + public partial class MySqlStreamStore : StreamStoreBase { private readonly MySqlStreamStoreSettings _settings; private readonly Func _createConnection; diff --git a/src/SqlStreamStore.MySql/SqlStreamStore.MySql.csproj b/src/SqlStreamStore.MySql/SqlStreamStore.MySql.csproj index 4328836b4..4e3dc2820 100644 --- a/src/SqlStreamStore.MySql/SqlStreamStore.MySql.csproj +++ b/src/SqlStreamStore.MySql/SqlStreamStore.MySql.csproj @@ -3,7 +3,7 @@ João P. Bragança, Damian Hickey MySQL provider for SQL StreamStore Stream Store - MySQL - netstandard2.0 + netstandard2.0 SqlStreamStore.MySql SqlStreamStore.MySql SqlStreamStore @@ -12,7 +12,7 @@ - + diff --git a/src/SqlStreamStore.Postgres/PgSqlScriptsV1/AppendToStream.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/AppendToStream.sql new file mode 100644 index 000000000..7499a2601 --- /dev/null +++ b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/AppendToStream.sql @@ -0,0 +1,178 @@ +CREATE OR REPLACE FUNCTION __schema__.append_to_stream( + _stream_id CHAR(42), + _stream_id_original VARCHAR(1000), + _metadata_stream_id CHAR(42), + _expected_version INT, + _created_utc TIMESTAMP, + _new_stream_messages __schema__.new_stream_message []) + RETURNS TABLE( + current_version INT, + current_position BIGINT + ) AS $F$ +DECLARE + _current_version INT; + _current_position BIGINT; + _stream_id_internal INT; + _success INT; + _max_age INT; + _max_count INT; +BEGIN + IF _created_utc IS NULL + THEN + _created_utc = now() at time zone 'utc'; + END IF; + + IF _expected_version < 0 + THEN + SELECT __schema__.messages.json_data :: JSON->>'MaxAge', __schema__.messages.json_data :: JSON->>'MaxCount' + INTO _max_age, _max_count + FROM __schema__.messages + JOIN __schema__.streams ON __schema__.messages.stream_id_internal = __schema__.streams.id_internal + WHERE __schema__.streams.id = _metadata_stream_id + ORDER BY __schema__.messages.stream_version DESC + LIMIT 1; + + INSERT INTO __schema__.streams (id, id_original, max_age, max_count) + SELECT _stream_id, _stream_id_original, _max_age, _max_count + ON CONFLICT DO NOTHING; + GET DIAGNOSTICS _success = ROW_COUNT; + + END IF; + + IF _expected_version = -1 /* ExpectedVersion.Empty */ + THEN + + IF _success = 0 AND + cardinality(_new_stream_messages) > 0 AND + (SELECT __schema__.streams.version FROM __schema__.streams WHERE __schema__.streams.id_internal = _stream_id_internal) > 0 + THEN + + RAISE EXCEPTION 'WrongExpectedVersion'; + END IF; + ELSIF _expected_version = -3 /* ExpectedVersion.NoStream */ + THEN + IF _success = 0 AND cardinality(_new_stream_messages) > 0 + THEN + PERFORM __schema__.enforce_idempotent_append( + _stream_id, + 0, + false, + _new_stream_messages); + SELECT version, position, id_internal + INTO _current_version, _current_position, _stream_id_internal + FROM __schema__.streams + WHERE __schema__.streams.id = _stream_id; + + RETURN QUERY + SELECT _current_version, _current_position; + RETURN; + END IF; + END IF; + + SELECT (CASE _expected_version + WHEN -2 /* ExpectedVersion.Any */ + THEN coalesce( + __schema__.read_stream_version_of_message_id( + __schema__.streams.id_internal, + _new_stream_messages [ 1 ].message_id) - 1, + __schema__.streams.version) + WHEN -3 THEN -1 /* ExpectedVersion.NoStream */ + WHEN -1 THEN -1 /* ExpectedVersion.Empty */ + ELSE _expected_version END), __schema__.streams.position, __schema__.streams.id_internal + INTO _current_version, _current_position, _stream_id_internal + FROM __schema__.streams + WHERE __schema__.streams.id = _stream_id; + + IF (_expected_version >= 0 AND ( + SELECT __schema__.streams.version + FROM __schema__.streams + WHERE __schema__.streams.id_internal = _stream_id_internal + ) < _expected_version) + THEN + RAISE EXCEPTION 'WrongExpectedVersion'; + END IF; + + IF (_expected_version >= 0 AND _stream_id_internal IS NULL) + THEN + RAISE EXCEPTION 'WrongExpectedVersion'; + END IF; + + IF cardinality(_new_stream_messages) > 0 + THEN + INSERT INTO __schema__.messages (message_id, + stream_id_internal, + stream_version, + created_utc, + type, + json_data, + json_metadata, + tx_id) + SELECT m.message_id, _stream_id_internal, _current_version + (row_number() + over ()) :: int, _created_utc, m.type, m.json_data, m.json_metadata, txid_current() + FROM unnest(_new_stream_messages) m + ON CONFLICT DO NOTHING; + GET DIAGNOSTICS _success = ROW_COUNT; + + IF (_success <> cardinality(_new_stream_messages)) + THEN + IF (_expected_version = -2) /* ExpectedVersion.Any */ + THEN + + PERFORM __schema__.enforce_idempotent_append( + _stream_id, + _current_version + 1 - _success, + false, + _new_stream_messages); + + ELSEIF _expected_version = -3 /* ExpectedVersion.NoStream */ + THEN + RAISE EXCEPTION 'WhyAreYouHere'; /* there is no way to get here? */ + ELSEIF _expected_version = -1 /* ExpectedVersion.Empty */ + THEN + PERFORM __schema__.enforce_idempotent_append( + _stream_id, + 0, + false, + _new_stream_messages); + ELSE + PERFORM __schema__.enforce_idempotent_append( + _stream_id, + _expected_version + 1 - _success, + true, + _new_stream_messages); + SELECT version, position, id_internal + INTO _current_version, _current_position, _stream_id_internal + FROM __schema__.streams + WHERE __schema__.streams.id = _stream_id; + + RETURN QUERY + SELECT _current_version, _current_position; + RETURN; + END IF; + END IF; + + SELECT COALESCE(__schema__.messages.position, -1), COALESCE(__schema__.messages.stream_version, -1) + INTO _current_position, _current_version + FROM __schema__.messages + WHERE __schema__.messages.stream_id_internal = _stream_id_internal + ORDER BY __schema__.messages.position DESC + LIMIT 1; + + UPDATE __schema__.streams + SET "version" = _current_version, + "position" = _current_position + WHERE id_internal = _stream_id_internal; + + RETURN QUERY + SELECT _current_version, _current_position; + + ELSE + RETURN QUERY + SELECT -1, -1 :: BIGINT; + + END IF; + +END; + +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/DeleteStream.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/DeleteStream.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/DeleteStream.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/DeleteStream.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/DeleteStreamMessages.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/DeleteStreamMessages.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/DeleteStreamMessages.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/DeleteStreamMessages.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/DropAll.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/DropAll.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/DropAll.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/DropAll.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/EnableExplainAnalyze.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/EnableExplainAnalyze.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/EnableExplainAnalyze.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/EnableExplainAnalyze.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/EnforceIdempotentAppend.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/EnforceIdempotentAppend.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/EnforceIdempotentAppend.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/EnforceIdempotentAppend.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/ListStreams.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/ListStreams.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/ListStreams.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/ListStreams.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/ListStreamsEndingWith.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/ListStreamsEndingWith.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/ListStreamsEndingWith.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/ListStreamsEndingWith.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/ListStreamsStartingWith.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/ListStreamsStartingWith.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/ListStreamsStartingWith.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/ListStreamsStartingWith.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/Parameters.cs b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/Parameters.cs similarity index 99% rename from src/SqlStreamStore.Postgres/PgSqlScripts/Parameters.cs rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/Parameters.cs index 5659a08e9..aa621cb8e 100644 --- a/src/SqlStreamStore.Postgres/PgSqlScripts/Parameters.cs +++ b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/Parameters.cs @@ -1,4 +1,4 @@ -namespace SqlStreamStore.PgSqlScripts +namespace SqlStreamStore.PgSqlScriptsV1 { using System; using Npgsql; diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/Read.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/Read.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/Read.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/Read.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScriptsV1/ReadAll.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/ReadAll.sql new file mode 100644 index 000000000..cca4f083b --- /dev/null +++ b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/ReadAll.sql @@ -0,0 +1,64 @@ +CREATE OR REPLACE FUNCTION __schema__.read_all2( + _count INT, + _position BIGINT, + _forwards BOOLEAN, + _prefetch BOOLEAN, + _max_position BIGINT +) + -- RETURNS TABLE( + -- stream_id VARCHAR(1000), + -- message_id UUID, + -- stream_version INT, + -- "position" BIGINT, + -- create_utc TIMESTAMP, + -- "type" VARCHAR(128), + -- json_metadata JSONB, + -- json_data JSONB, + -- max_age INT + -- ) + RETURNS SETOF REFCURSOR + +AS $F$ + +DECLARE + -- _stream_id_internal INT; + -- _stream_info REFCURSOR := 'stream_info'; + _messages REFCURSOR := 'messages'; + _txinfo REFCURSOR := 'tx_info'; + +BEGIN + + OPEN _messages FOR + WITH messages AS ( + SELECT __schema__.streams.id_original, + __schema__.messages.message_id, + __schema__.messages.stream_version, + __schema__.messages.position, + __schema__.messages.created_utc, + __schema__.messages.type, + __schema__.messages.json_metadata, + (CASE _prefetch + WHEN TRUE THEN __schema__.messages.json_data + ELSE NULL END), + __schema__.streams.max_age + FROM __schema__.messages + INNER JOIN __schema__.streams ON __schema__.messages.stream_id_internal = __schema__.streams.id_internal + WHERE (CASE + WHEN _forwards AND _max_position > 0 THEN __schema__.messages.position >= _position AND __schema__.messages.position <= _max_position + WHEN _forwards THEN __schema__.messages.position >= _position + ELSE __schema__.messages.position <= _position END) + ORDER BY + (CASE WHEN _forwards THEN __schema__.messages.position END), + (CASE WHEN not _forwards THEN __schema__.messages.position END) DESC + LIMIT _count + ) + SELECT * FROM messages LIMIT _count; + + RETURN NEXT _messages; + + OPEN _txinfo FOR + SELECT txid_current_snapshot()::TEXT; + RETURN NEXT _txinfo; +END; +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/ReadHeadPosition.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/ReadHeadPosition.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/ReadHeadPosition.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/ReadHeadPosition.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/ReadJsonData.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/ReadJsonData.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/ReadJsonData.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/ReadJsonData.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/ReadSchemaVersion.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/ReadSchemaVersion.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/ReadSchemaVersion.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/ReadSchemaVersion.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/ReadStreamHeadPosition.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/ReadStreamHeadPosition.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/ReadStreamHeadPosition.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/ReadStreamHeadPosition.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/ReadStreamHeadVersion.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/ReadStreamHeadVersion.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/ReadStreamHeadVersion.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/ReadStreamHeadVersion.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/ReadStreamVersionOfMessageId.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/ReadStreamVersionOfMessageId.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/ReadStreamVersionOfMessageId.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/ReadStreamVersionOfMessageId.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/Scavenge.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/Scavenge.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/Scavenge.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/Scavenge.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScriptsV1/Schema.cs b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/Schema.cs new file mode 100644 index 000000000..2f629aa87 --- /dev/null +++ b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/Schema.cs @@ -0,0 +1,39 @@ +namespace SqlStreamStore.PgSqlScriptsV1 +{ + internal class Schema + { + private readonly string _schema; + private readonly Scripts _scripts; + + public string Definition => _scripts.CreateSchema; + public string DropAll => _scripts.DropAll; + public string EnableExplainAnalyze => _scripts.EnableExplainAnalyze; + + public string NewStreamMessage => $"{_schema}.new_stream_message"; + + public string AppendToStream => $"{_schema}.append_to_stream"; + public string Scavenge => $"{_schema}.scavenge"; + public string SetStreamMetadata => $"{_schema}.set_stream_metadata"; + public string DeleteStream => $"{_schema}.delete_stream"; + public string DeleteStreamMessages => $"{_schema}.delete_stream_messages"; + public string ListStreams => $"{_schema}.list_streams"; + public string ListStreamsStartingWith => $"{_schema}.list_streams_starting_with"; + public string ListStreamsEndingWith => $"{_schema}.list_streams_ending_with"; + public string Read => $"{_schema}.read"; + public string ReadAll => $"{_schema}.read_all2"; + public string ReadAllHeadPosition => $"{_schema}.read_head_position"; + public string ReadStreamHeadPosition => $"{_schema}.read_stream_head_position"; + public string ReadStreamHeadVersion => $"{_schema}.read_stream_head_version"; + public string ReadJsonData => $"{_schema}.read_json_data"; + public string ReadSchemaVersion => $"{_schema}.read_schema_version"; + public string ReadStreamMessageBeforeCreatedCount => $"{_schema}.read_stream_message_before_created_count"; + + public static string FetchAll(string refcursor) => $@"FETCH ALL IN ""{refcursor}"";"; + + public Schema(string schema) + { + _schema = schema; + _scripts = new Scripts(schema); + } + } +} \ No newline at end of file diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/Scripts.cs b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/Scripts.cs similarity index 98% rename from src/SqlStreamStore.Postgres/PgSqlScripts/Scripts.cs rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/Scripts.cs index c134ae440..757bd8416 100644 --- a/src/SqlStreamStore.Postgres/PgSqlScripts/Scripts.cs +++ b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/Scripts.cs @@ -1,4 +1,4 @@ -namespace SqlStreamStore.PgSqlScripts +namespace SqlStreamStore.PgSqlScriptsV1 { using System; using System.Collections.Concurrent; diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/SetStreamMetadata.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/SetStreamMetadata.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/SetStreamMetadata.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/SetStreamMetadata.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScriptsV1/Tables.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/Tables.sql new file mode 100644 index 000000000..e7eb520cf --- /dev/null +++ b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/Tables.sql @@ -0,0 +1,63 @@ +CREATE SEQUENCE IF NOT EXISTS __schema__.streams_seq + START 1; + +CREATE TABLE IF NOT EXISTS __schema__.streams ( + id CHAR(42) NOT NULL, + id_original VARCHAR(1000) NOT NULL, + id_internal INT NOT NULL DEFAULT nextval('__schema__.streams_seq'), + version INT NOT NULL DEFAULT (-1), + position BIGINT NOT NULL DEFAULT (-1), + max_age INT NULL, + max_count INT NULL, + CONSTRAINT pk_streams PRIMARY KEY (id_internal), + CONSTRAINT uq_streams_id UNIQUE (id), + CONSTRAINT ck_version_gte_negative_one CHECK (version >= -1) +); + +CREATE INDEX IF NOT EXISTS ix_id_original + ON __schema__.streams (id_original); + +CREATE INDEX IF NOT EXISTS ix_id_original_reversed + ON __schema__.streams (REVERSE(id_original)); + +COMMENT ON SCHEMA __schema__ +IS '{ "version": 1 }'; + +ALTER SEQUENCE __schema__.streams_seq + OWNED BY __schema__.streams.id_internal; + +CREATE SEQUENCE IF NOT EXISTS __schema__.messages_seq + START 0 + MINVALUE 0; + +CREATE TABLE IF NOT EXISTS __schema__.messages ( + stream_id_internal INT NOT NULL, + stream_version INT NOT NULL, + "position" BIGINT NOT NULL DEFAULT nextval('__schema__.messages_seq'), + message_id UUID NOT NULL, + created_utc TIMESTAMP NOT NULL, + type VARCHAR(128) NOT NULL, + json_data JSONB NOT NULL, + json_metadata JSONB, + tx_id BIGINT NOT NULL DEFAULT 0, + CONSTRAINT pk_messages PRIMARY KEY (position), + CONSTRAINT fk_messages_stream FOREIGN KEY (stream_id_internal) REFERENCES __schema__.streams (id_internal), + CONSTRAINT uq_messages_stream_id_internal_and_stream_version UNIQUE (stream_id_internal, stream_version), + CONSTRAINT uq_stream_id_internal_and_message_id UNIQUE (stream_id_internal, message_id), + CONSTRAINT ck_stream_version_gte_zero CHECK (stream_version >= 0) +); + +ALTER SEQUENCE __schema__.messages_seq + OWNED BY __schema__.messages.position; + +DO $F$ +BEGIN + CREATE TYPE __schema__.new_stream_message AS ( + message_id UUID, + "type" VARCHAR(128), + json_data JSONB, + json_metadata JSONB); + EXCEPTION + WHEN duplicate_object + THEN null; +END $F$; \ No newline at end of file diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/AppendToStream.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/AppendToStream.sql similarity index 99% rename from src/SqlStreamStore.Postgres/PgSqlScripts/AppendToStream.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV2/AppendToStream.sql index d54a1a366..745a390af 100644 --- a/src/SqlStreamStore.Postgres/PgSqlScripts/AppendToStream.sql +++ b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/AppendToStream.sql @@ -3,7 +3,7 @@ CREATE OR REPLACE FUNCTION __schema__.append_to_stream( _stream_id_original VARCHAR(1000), _metadata_stream_id CHAR(42), _expected_version INT, - _created_utc TIMESTAMP, + _created_utc TIMESTAMP WITH TIME ZONE, _new_stream_messages __schema__.new_stream_message []) RETURNS TABLE( current_version INT, diff --git a/src/SqlStreamStore.Postgres/PgSqlScriptsV2/DeleteStream.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/DeleteStream.sql new file mode 100644 index 000000000..9ec69ee94 --- /dev/null +++ b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/DeleteStream.sql @@ -0,0 +1,73 @@ +CREATE OR REPLACE FUNCTION __schema__.delete_stream( + _stream_id CHAR(42), + _expected_version INT, + _created_utc TIMESTAMP WITH TIME ZONE, + _deletion_tracking_disabled BOOLEAN, + _deleted_stream_id CHAR(42), + _deleted_stream_id_original VARCHAR(1000), + _deleted_stream_message __schema__.new_stream_message) + RETURNS VOID +AS $F$ +DECLARE + _stream_id_internal INT; + _latest_stream_version INT; + _affected INT; +BEGIN + IF _created_utc IS NULL + THEN + _created_utc = now() at time zone 'utc'; + END IF; + SELECT __schema__.streams.id_internal + INTO _stream_id_internal + FROM __schema__.streams + WHERE __schema__.streams.id = _stream_id; + + IF _expected_version = -1 /* ExpectedVersion.NoStream */ + THEN + RAISE EXCEPTION 'WrongExpectedVersion'; + ELSIF _expected_version >= 0 /* ExpectedVersion */ + THEN + IF _stream_id_internal IS NULL + THEN + RAISE EXCEPTION 'WrongExpectedVersion'; + END IF; + + SELECT __schema__.messages.stream_version + INTO _latest_stream_version + FROM __schema__.messages + WHERE __schema__.messages.stream_id_internal = _stream_id_internal + ORDER BY __schema__.messages.position DESC + LIMIT 1; + + IF _latest_stream_version != _expected_version + THEN + RAISE EXCEPTION 'WrongExpectedVersion'; + END IF; + END IF; + + DELETE FROM __schema__.messages WHERE __schema__.messages.stream_id_internal = _stream_id_internal; + + DELETE FROM __schema__.streams WHERE __schema__.streams.id = _stream_id; + + IF (_deletion_tracking_disabled = TRUE) + THEN + RETURN; + END IF; + + GET DIAGNOSTICS _affected = ROW_COUNT; + + IF _affected > 0 + THEN + PERFORM __schema__.append_to_stream( + _deleted_stream_id, + _deleted_stream_id_original, + NULL, + -2, + _created_utc, + ARRAY [_deleted_stream_message]); + + END IF; +END; + +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.Postgres/PgSqlScriptsV2/DeleteStreamMessages.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/DeleteStreamMessages.sql new file mode 100644 index 000000000..1ad2b0861 --- /dev/null +++ b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/DeleteStreamMessages.sql @@ -0,0 +1,46 @@ +CREATE OR REPLACE FUNCTION __schema__.delete_stream_messages( + _stream_id CHAR(42), + _message_ids UUID [], + _deletion_tracking_disabled BOOLEAN, + _deleted_stream_id CHAR(42), + _deleted_stream_id_original VARCHAR(1000), + _created_utc TIMESTAMP WITH TIME ZONE, + _deleted_messages __schema__.new_stream_message [] +) + RETURNS VOID +AS $F$ +DECLARE + _stream_id_internal INT; + _deleted_count NUMERIC; +BEGIN + IF _created_utc IS NULL THEN + _created_utc = now() at time zone 'utc'; + END IF; + + SELECT __schema__.streams.id_internal + INTO _stream_id_internal + FROM __schema__.streams + WHERE __schema__.streams.id = _stream_id; + + WITH deleted AS (DELETE FROM __schema__.messages + WHERE __schema__.messages.stream_id_internal = _stream_id_internal + AND __schema__.messages.message_id = ANY (_message_ids) + RETURNING *) + SELECT count(*) + FROM deleted + INTO _deleted_count; + + IF (_deletion_tracking_disabled = FALSE AND _deleted_count > 0) + THEN + PERFORM __schema__.append_to_stream( + _deleted_stream_id, + _deleted_stream_id_original, + NULL, + -2, + _created_utc, + _deleted_messages); + END IF; +END; + +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.Postgres/PgSqlScriptsV2/DropAll.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/DropAll.sql new file mode 100644 index 000000000..aa6c25354 --- /dev/null +++ b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/DropAll.sql @@ -0,0 +1,78 @@ +DROP FUNCTION IF EXISTS __schema__.append_to_stream( + CHAR(42), + VARCHAR(1000), + INT, + TIMESTAMP WITH TIME ZONE, + __schema__ .new_stream_message [] +) CASCADE; +DROP FUNCTION IF EXISTS __schema__.delete_stream( + CHAR(42), + INT, + TIMESTAMP WITH TIME ZONE, + CHAR(42), + VARCHAR(1000), + __schema__ .new_stream_message +) CASCADE; +DROP FUNCTION IF EXISTS __schema__.delete_stream_messages( + CHAR(42), + UUID [], + CHAR(42), + VARCHAR(1000), + TIMESTAMP WITH TIME ZONE, + __schema__ .new_stream_message [] +) CASCADE; +DROP FUNCTION IF EXISTS __schema__.enforce_idempotent_append( + CHAR(42), + INT, + BOOLEAN, + __schema__ .new_stream_message [] +) CASCADE; +DROP FUNCTION IF EXISTS __schema__.list_streams( + INT, + INT +) CASCADE; +DROP FUNCTION IF EXISTS __schema__.list_streams_starting_with( + VARCHAR(1000), + INT, + INT +) CASCADE; +DROP FUNCTION IF EXISTS __schema__.list_streams_ending_with( + VARCHAR(1000), + INT, + INT +) CASCADE; +DROP FUNCTION IF EXISTS __schema__.read( + CHAR(42), + INT, + INT, + BOOLEAN, + BOOLEAN +) CASCADE; +DROP FUNCTION IF EXISTS __schema__.read_all( + INT, + BIGINT, + BOOLEAN, + BOOLEAN +) CASCADE; +DROP FUNCTION IF EXISTS __schema__.read_schema_version() CASCADE; +DROP FUNCTION IF EXISTS __schema__.read_head_position() CASCADE; +DROP FUNCTION IF EXISTS __schema__.read_json_data( + CHAR(42), + INT +) CASCADE; +DROP FUNCTION IF EXISTS __schema__.read_stream_message_before_created_count( + CHAR(42), + TIMESTAMP WITH TIME ZONE +) CASCADE; +DROP FUNCTION IF EXISTS __schema__.read_stream_version_of_message_id( + INT, + UUID +); +DROP FUNCTION IF EXISTS __schema__.scavenge( + CHAR(42) +) CASCADE; +DROP TABLE IF EXISTS __schema__.messages CASCADE; +DROP SEQUENCE IF EXISTS __schema__.messages_seq CASCADE; +DROP TABLE IF EXISTS __schema__.streams CASCADE; +DROP SEQUENCE IF EXISTS __schema__.streams_seq CASCADE; +DROP TYPE IF EXISTS __schema__.new_stream_message CASCADE; \ No newline at end of file diff --git a/src/SqlStreamStore.Postgres/PgSqlScriptsV2/EnableExplainAnalyze.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/EnableExplainAnalyze.sql new file mode 100644 index 000000000..450617841 --- /dev/null +++ b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/EnableExplainAnalyze.sql @@ -0,0 +1,7 @@ +LOAD 'auto_explain'; +SET auto_explain.log_min_duration = 0; +SET auto_explain.log_nested_statements = ON; +SET auto_explain.log_analyze = ON; +SET auto_explain.log_format = TEXT; +SET auto_explain.log_buffers = ON; +SET auto_explain.log_verbose = ON; diff --git a/src/SqlStreamStore.Postgres/PgSqlScriptsV2/EnforceIdempotentAppend.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/EnforceIdempotentAppend.sql new file mode 100644 index 000000000..e0602a24e --- /dev/null +++ b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/EnforceIdempotentAppend.sql @@ -0,0 +1,38 @@ +CREATE OR REPLACE FUNCTION __schema__.enforce_idempotent_append( + _stream_id CHAR(42), + _start INT, + _check_length BOOLEAN, + _new_stream_messages __schema__.new_stream_message []) + RETURNS VOID AS $F$ +DECLARE + _message_id_record RECORD; + _message_id_cursor REFCURSOR; + _message_ids UUID [] = '{}' :: UUID []; +BEGIN + _message_id_cursor = (SELECT * + FROM __schema__.read(_stream_id, cardinality(_new_stream_messages), _start, true, false) + OFFSET 1); + + FETCH FROM _message_id_cursor + INTO _message_id_record; + + WHILE FOUND LOOP + _message_ids = array_append(_message_ids, _message_id_record.message_id); + + FETCH FROM _message_id_cursor + INTO _message_id_record; + END LOOP; + + IF (_check_length AND cardinality(_new_stream_messages) > cardinality(_message_ids)) + THEN + RAISE EXCEPTION 'WrongExpectedVersion' + USING HINT = 'Wrong message count'; + END IF; + + IF _message_ids <> (SELECT ARRAY(SELECT n.message_id FROM unnest(_new_stream_messages) n)) + THEN + RAISE EXCEPTION 'WrongExpectedVersion'; + END IF; +END; +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ListStreams.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ListStreams.sql new file mode 100644 index 000000000..22a2c0650 --- /dev/null +++ b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ListStreams.sql @@ -0,0 +1,19 @@ +CREATE OR REPLACE FUNCTION __schema__.list_streams( + _max_count INT, + _after_id_internal INT +) + RETURNS TABLE( + stream_id VARCHAR(1000), + id_internal INT + ) AS $F$ +BEGIN + RETURN QUERY + SELECT __schema__.streams.id_original, __schema__.streams.id_internal + FROM __schema__.streams + WHERE __schema__.streams.id_internal > _after_id_internal + ORDER BY __schema__.streams.id_internal ASC + LIMIT _max_count; +END; + +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ListStreamsEndingWith.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ListStreamsEndingWith.sql new file mode 100644 index 000000000..b9e6322d8 --- /dev/null +++ b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ListStreamsEndingWith.sql @@ -0,0 +1,21 @@ +CREATE OR REPLACE FUNCTION __schema__.list_streams_ending_with( + _pattern VARCHAR(1000), + _max_count INT, + _after_id_internal INT +) + RETURNS TABLE( + stream_id VARCHAR(1000), + id_internal INT + ) AS $F$ +BEGIN + RETURN QUERY + SELECT __schema__.streams.id_original, __schema__.streams.id_internal + FROM __schema__.streams + WHERE REVERSE(__schema__.streams.id_original) LIKE CONCAT(REVERSE(_pattern), '%') + AND __schema__.streams.id_internal > _after_id_internal + ORDER BY __schema__.streams.id_internal ASC + LIMIT _max_count; +END; + +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ListStreamsStartingWith.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ListStreamsStartingWith.sql new file mode 100644 index 000000000..4a93e5cd8 --- /dev/null +++ b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ListStreamsStartingWith.sql @@ -0,0 +1,21 @@ +CREATE OR REPLACE FUNCTION __schema__.list_streams_starting_with( + _pattern VARCHAR(1000), + _max_count INT, + _after_id_internal INT +) + RETURNS TABLE( + stream_id VARCHAR(1000), + id_internal INT + ) AS $F$ +BEGIN + RETURN QUERY + SELECT __schema__.streams.id_original, __schema__.streams.id_internal + FROM __schema__.streams + WHERE __schema__.streams.id_original LIKE CONCAT(_pattern, '%') + AND __schema__.streams.id_internal > _after_id_internal + ORDER BY __schema__.streams.id_internal ASC + LIMIT _max_count; +END; + +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.Postgres/PgSqlScriptsV2/Migration.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/Migration.sql new file mode 100644 index 000000000..914dcbcad --- /dev/null +++ b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/Migration.sql @@ -0,0 +1,391 @@ +SET TimeZone='UTC'; +ALTER TABLE __schema__.messages ALTER COLUMN "created_utc" TYPE TIMESTAMP WITH TIME ZONE; +COMMENT ON SCHEMA __schema__ IS '{ "version": 2 }'; + +CREATE OR REPLACE FUNCTION __schema__.append_to_stream( + _stream_id CHAR(42), + _stream_id_original VARCHAR(1000), + _metadata_stream_id CHAR(42), + _expected_version INT, + _created_utc TIMESTAMP WITH TIME ZONE, + _new_stream_messages __schema__.new_stream_message []) + RETURNS TABLE( + current_version INT, + current_position BIGINT + ) AS $F$ +DECLARE + _current_version INT; + _current_position BIGINT; + _stream_id_internal INT; + _success INT; + _max_age INT; + _max_count INT; +BEGIN + IF _created_utc IS NULL + THEN + _created_utc = now() at time zone 'utc'; + END IF; + + IF _expected_version < 0 + THEN + SELECT __schema__.messages.json_data :: JSON->>'MaxAge', __schema__.messages.json_data :: JSON->>'MaxCount' + INTO _max_age, _max_count + FROM __schema__.messages + JOIN __schema__.streams ON __schema__.messages.stream_id_internal = __schema__.streams.id_internal + WHERE __schema__.streams.id = _metadata_stream_id + ORDER BY __schema__.messages.stream_version DESC + LIMIT 1; + + INSERT INTO __schema__.streams (id, id_original, max_age, max_count) + SELECT _stream_id, _stream_id_original, _max_age, _max_count + ON CONFLICT DO NOTHING; + GET DIAGNOSTICS _success = ROW_COUNT; + + END IF; + + IF _expected_version = -1 /* ExpectedVersion.Empty */ + THEN + + IF _success = 0 AND + cardinality(_new_stream_messages) > 0 AND + (SELECT __schema__.streams.version FROM __schema__.streams WHERE __schema__.streams.id_internal = _stream_id_internal) > 0 + THEN + + RAISE EXCEPTION 'WrongExpectedVersion'; + END IF; + ELSIF _expected_version = -3 /* ExpectedVersion.NoStream */ + THEN + IF _success = 0 AND cardinality(_new_stream_messages) > 0 + THEN + PERFORM __schema__.enforce_idempotent_append( + _stream_id, + 0, + false, + _new_stream_messages); + SELECT version, position, id_internal + INTO _current_version, _current_position, _stream_id_internal + FROM __schema__.streams + WHERE __schema__.streams.id = _stream_id; + + RETURN QUERY + SELECT _current_version, _current_position; + RETURN; + END IF; + END IF; + + SELECT (CASE _expected_version + WHEN -2 /* ExpectedVersion.Any */ + THEN coalesce( + __schema__.read_stream_version_of_message_id( + __schema__.streams.id_internal, + _new_stream_messages [ 1 ].message_id) - 1, + __schema__.streams.version) + WHEN -3 THEN -1 /* ExpectedVersion.NoStream */ + WHEN -1 THEN -1 /* ExpectedVersion.Empty */ + ELSE _expected_version END), __schema__.streams.position, __schema__.streams.id_internal + INTO _current_version, _current_position, _stream_id_internal + FROM __schema__.streams + WHERE __schema__.streams.id = _stream_id; + + IF (_expected_version >= 0 AND ( + SELECT __schema__.streams.version + FROM __schema__.streams + WHERE __schema__.streams.id_internal = _stream_id_internal + ) < _expected_version) + THEN + RAISE EXCEPTION 'WrongExpectedVersion'; + END IF; + + IF (_expected_version >= 0 AND _stream_id_internal IS NULL) + THEN + RAISE EXCEPTION 'WrongExpectedVersion'; + END IF; + + IF cardinality(_new_stream_messages) > 0 + THEN + INSERT INTO __schema__.messages (message_id, + stream_id_internal, + stream_version, + created_utc, + type, + json_data, + json_metadata) + SELECT m.message_id, _stream_id_internal, _current_version + (row_number() + over ()) :: int, _created_utc, m.type, m.json_data, m.json_metadata + FROM unnest(_new_stream_messages) m + ON CONFLICT DO NOTHING; + GET DIAGNOSTICS _success = ROW_COUNT; + + IF (_success <> cardinality(_new_stream_messages)) + THEN + IF (_expected_version = -2) /* ExpectedVersion.Any */ + THEN + + PERFORM __schema__.enforce_idempotent_append( + _stream_id, + _current_version + 1 - _success, + false, + _new_stream_messages); + + ELSEIF _expected_version = -3 /* ExpectedVersion.NoStream */ + THEN + RAISE EXCEPTION 'WhyAreYouHere'; /* there is no way to get here? */ + ELSEIF _expected_version = -1 /* ExpectedVersion.Empty */ + THEN + PERFORM __schema__.enforce_idempotent_append( + _stream_id, + 0, + false, + _new_stream_messages); + ELSE + PERFORM __schema__.enforce_idempotent_append( + _stream_id, + _expected_version + 1 - _success, + true, + _new_stream_messages); + SELECT version, position, id_internal + INTO _current_version, _current_position, _stream_id_internal + FROM __schema__.streams + WHERE __schema__.streams.id = _stream_id; + + RETURN QUERY + SELECT _current_version, _current_position; + RETURN; + END IF; + END IF; + + SELECT COALESCE(__schema__.messages.position, -1), COALESCE(__schema__.messages.stream_version, -1) + INTO _current_position, _current_version + FROM __schema__.messages + WHERE __schema__.messages.stream_id_internal = _stream_id_internal + ORDER BY __schema__.messages.position DESC + LIMIT 1; + + UPDATE __schema__.streams + SET "version" = _current_version, + "position" = _current_position + WHERE id_internal = _stream_id_internal; + + RETURN QUERY + SELECT _current_version, _current_position; + + ELSE + RETURN QUERY + SELECT -1, -1 :: BIGINT; + + END IF; + +END; + +$F$ +LANGUAGE 'plpgsql'; + + +CREATE OR REPLACE FUNCTION __schema__.delete_stream( + _stream_id CHAR(42), + _expected_version INT, + _created_utc TIMESTAMP WITH TIME ZONE, + _deletion_tracking_disabled BOOLEAN, + _deleted_stream_id CHAR(42), + _deleted_stream_id_original VARCHAR(1000), + _deleted_stream_message __schema__.new_stream_message) + RETURNS VOID +AS $F$ +DECLARE + _stream_id_internal INT; + _latest_stream_version INT; + _affected INT; +BEGIN + IF _created_utc IS NULL + THEN + _created_utc = now() at time zone 'utc'; + END IF; + SELECT __schema__.streams.id_internal + INTO _stream_id_internal + FROM __schema__.streams + WHERE __schema__.streams.id = _stream_id; + + IF _expected_version = -1 /* ExpectedVersion.NoStream */ + THEN + RAISE EXCEPTION 'WrongExpectedVersion'; + ELSIF _expected_version >= 0 /* ExpectedVersion */ + THEN + IF _stream_id_internal IS NULL + THEN + RAISE EXCEPTION 'WrongExpectedVersion'; + END IF; + + SELECT __schema__.messages.stream_version + INTO _latest_stream_version + FROM __schema__.messages + WHERE __schema__.messages.stream_id_internal = _stream_id_internal + ORDER BY __schema__.messages.position DESC + LIMIT 1; + + IF _latest_stream_version != _expected_version + THEN + RAISE EXCEPTION 'WrongExpectedVersion'; + END IF; + END IF; + + DELETE FROM __schema__.messages WHERE __schema__.messages.stream_id_internal = _stream_id_internal; + + DELETE FROM __schema__.streams WHERE __schema__.streams.id = _stream_id; + + IF (_deletion_tracking_disabled = TRUE) + THEN + RETURN; + END IF; + + GET DIAGNOSTICS _affected = ROW_COUNT; + + IF _affected > 0 + THEN + PERFORM __schema__.append_to_stream( + _deleted_stream_id, + _deleted_stream_id_original, + NULL, + -2, + _created_utc, + ARRAY [_deleted_stream_message]); + + END IF; +END; + +$F$ +LANGUAGE 'plpgsql'; + +CREATE OR REPLACE FUNCTION __schema__.delete_stream_messages( + _stream_id CHAR(42), + _message_ids UUID [], + _deletion_tracking_disabled BOOLEAN, + _deleted_stream_id CHAR(42), + _deleted_stream_id_original VARCHAR(1000), + _created_utc TIMESTAMP WITH TIME ZONE, + _deleted_messages __schema__.new_stream_message [] +) + RETURNS VOID +AS $F$ +DECLARE + _stream_id_internal INT; + _deleted_count NUMERIC; +BEGIN + IF _created_utc IS NULL THEN + _created_utc = now() at time zone 'utc'; + END IF; + + SELECT __schema__.streams.id_internal + INTO _stream_id_internal + FROM __schema__.streams + WHERE __schema__.streams.id = _stream_id; + + WITH deleted AS (DELETE FROM __schema__.messages + WHERE __schema__.messages.stream_id_internal = _stream_id_internal + AND __schema__.messages.message_id = ANY (_message_ids) + RETURNING *) + SELECT count(*) + FROM deleted + INTO _deleted_count; + + IF (_deletion_tracking_disabled = FALSE AND _deleted_count > 0) + THEN + PERFORM __schema__.append_to_stream( + _deleted_stream_id, + _deleted_stream_id_original, + NULL, + -2, + _created_utc, + _deleted_messages); + END IF; +END; + +$F$ +LANGUAGE 'plpgsql'; + +DROP FUNCTION public.read_all(int4, int8, bool, bool); +CREATE OR REPLACE FUNCTION __schema__.read_all( + _count INT, + _position BIGINT, + _forwards BOOLEAN, + _prefetch BOOLEAN +) + RETURNS TABLE( + stream_id VARCHAR(1000), + message_id UUID, + stream_version INT, + "position" BIGINT, + create_utc TIMESTAMP WITH TIME ZONE, + "type" VARCHAR(128), + json_metadata JSONB, + json_data JSONB, + max_age INT + ) +AS $F$ +BEGIN + + RETURN QUERY + WITH messages AS ( + SELECT __schema__.streams.id_original, + __schema__.messages.message_id, + __schema__.messages.stream_version, + __schema__.messages.position, + __schema__.messages.created_utc, + __schema__.messages.type, + __schema__.messages.json_metadata, + (CASE _prefetch + WHEN TRUE THEN __schema__.messages.json_data + ELSE NULL END), + __schema__.streams.max_age + FROM __schema__.messages + INNER JOIN __schema__.streams ON __schema__.messages.stream_id_internal = __schema__.streams.id_internal + WHERE (CASE + WHEN _forwards THEN __schema__.messages.position >= _position + ELSE __schema__.messages.position <= _position END) + ORDER BY + (CASE WHEN _forwards THEN __schema__.messages.position END), + (CASE WHEN not _forwards THEN __schema__.messages.position END) DESC + LIMIT _count + ) + SELECT * FROM messages LIMIT _count; +END; +$F$ +LANGUAGE 'plpgsql'; + +CREATE OR REPLACE FUNCTION __schema__.set_stream_metadata( + _stream_id CHAR(42), + _metadata_stream_id CHAR(42), + _metadata_stream_id_original CHAR(42), + _max_age INT, + _max_count INT, + _expected_version INT, + _created_utc TIMESTAMP WITH TIME ZONE, + _metadata_message __schema__.new_stream_message) + RETURNS INT AS $F$ +DECLARE + _current_version INT; +BEGIN + IF _created_utc IS NULL + THEN + _created_utc = now() at time zone 'utc'; + END IF; + + SELECT current_version + FROM __schema__.append_to_stream( + _metadata_stream_id, + _metadata_stream_id_original, + NULL, + _expected_version, + _created_utc, + ARRAY [_metadata_message] + ) + INTO _current_version; + + UPDATE __schema__.streams + SET max_age = _max_age, + max_count = _max_count + WHERE __schema__.streams.id = _stream_id; + + RETURN _current_version; +END; +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.Postgres/PgSqlScriptsV2/Parameters.cs b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/Parameters.cs new file mode 100644 index 000000000..a77c065c5 --- /dev/null +++ b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/Parameters.cs @@ -0,0 +1,272 @@ +namespace SqlStreamStore.PgSqlScriptsV2 +{ + using System; + using Npgsql; + using NpgsqlTypes; + using SqlStreamStore.Infrastructure; + using SqlStreamStore.Streams; + using StreamStoreStore.Json; + + internal static class Parameters + { + private const int StreamIdSize = 42; + private const int OriginalStreamIdSize = 1000; + + public static NpgsqlParameter DeletedStreamId => new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Char, + Size = StreamIdSize, + TypedValue = PostgresqlStreamId.Deleted.Id + }; + + public static NpgsqlParameter DeletedStreamIdOriginal => new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Varchar, + Size = OriginalStreamIdSize, + TypedValue = PostgresqlStreamId.Deleted.IdOriginal + }; + + public static NpgsqlParameter StreamId(PostgresqlStreamId value) + { + return new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Char, + Size = StreamIdSize, + TypedValue = value.Id + }; + } + + public static NpgsqlParameter StreamIdOriginal(PostgresqlStreamId value) + { + return new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Varchar, + Size = OriginalStreamIdSize, + TypedValue = value.IdOriginal + }; + } + + public static NpgsqlParameter MetadataStreamId(PostgresqlStreamId value) + { + return new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Char, + Size = StreamIdSize, + TypedValue = value.Id + }; + } + + public static NpgsqlParameter MetadataStreamIdOriginal(PostgresqlStreamId value) + { + return new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Char, + Size = StreamIdSize, + TypedValue = value.IdOriginal + }; + } + + public static NpgsqlParameter DeletedMessages(PostgresqlStreamId streamId, params Guid[] messageIds) + { + return new NpgsqlParameter + { + TypedValue = Array.ConvertAll( + messageIds, + messageId => PostgresNewStreamMessage.FromNewStreamMessage( + Deleted.CreateMessageDeletedMessage(streamId.IdOriginal, messageId)) + ) + }; + } + + public static NpgsqlParameter DeletedStreamMessage(PostgresqlStreamId streamId) + { + return new NpgsqlParameter + { + TypedValue = PostgresNewStreamMessage.FromNewStreamMessage( + Deleted.CreateStreamDeletedMessage(streamId.IdOriginal)) + }; + } + + public static NpgsqlParameter ExpectedVersion(int value) + { + return new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Integer, + TypedValue = value + }; + } + + public static NpgsqlParameter CreatedUtc(DateTime? value) + { + return value.HasValue + ? (NpgsqlParameter) new NpgsqlParameter + { + TypedValue = value.Value, + NpgsqlDbType = NpgsqlDbType.TimestampTz + } + : new NpgsqlParameter + { + TypedValue = DBNull.Value, + NpgsqlDbType = NpgsqlDbType.TimestampTz + }; + } + + public static NpgsqlParameter NewStreamMessages(NewStreamMessage[] value) + { + return new NpgsqlParameter + { + TypedValue = Array.ConvertAll(value, PostgresNewStreamMessage.FromNewStreamMessage) + }; + } + + public static NpgsqlParameter MetadataStreamMessage( + PostgresqlStreamId streamId, + int expectedVersion, + MetadataMessage value) + { + var jsonData = SimpleJson.SerializeObject(value); + return new NpgsqlParameter + { + TypedValue = PostgresNewStreamMessage.FromNewStreamMessage( + new NewStreamMessage( + MetadataMessageIdGenerator.Create(streamId.IdOriginal, expectedVersion, jsonData), + "$stream-metadata", + jsonData)) + }; + } + + public static NpgsqlParameter Count(int value) + { + return new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Integer, + TypedValue = value + }; + } + + public static NpgsqlParameter Version(int value) + { + return new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Integer, + TypedValue = value + }; + } + + public static NpgsqlParameter ReadDirection(ReadDirection direction) + { + return new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Boolean, + TypedValue = direction == Streams.ReadDirection.Forward + }; + } + + public static NpgsqlParameter Prefetch(bool value) + { + return new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Boolean, + TypedValue = value + }; + } + + public static NpgsqlParameter MessageIds(Guid[] value) + { + return new NpgsqlParameter + { + Value = value + }; + } + + public static NpgsqlParameter Position(long value) + { + return new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Bigint, + TypedValue = value + }; + } + + public static NpgsqlParameter OptionalMaxAge(int? value) + { + return new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Integer, + NpgsqlValue = value.HasValue ? (object) value.Value : DBNull.Value + }; + } + + public static NpgsqlParameter OptionalMaxCount(int? value) + { + return new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Integer, + NpgsqlValue = value.HasValue ? (object) value.Value : DBNull.Value + }; + } + + public static NpgsqlParameter MaxCount(int value) + { + return new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Integer, + TypedValue = value + }; + } + + public static NpgsqlParameter OptionalStartingAt(int? value) + { + return value.HasValue + ? (NpgsqlParameter) new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Integer, + TypedValue = value.Value + } + : new NpgsqlParameter + { + TypedValue = DBNull.Value + }; + } + + public static NpgsqlParameter OptionalAfterIdInternal(int? value) + { + return value.HasValue + ? (NpgsqlParameter) new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Integer, + TypedValue = value.Value + } + : new NpgsqlParameter + { + TypedValue = DBNull.Value + }; + } + + public static NpgsqlParameter Pattern(string value) + { + return new NpgsqlParameter + { + TypedValue = value, + NpgsqlDbType = NpgsqlDbType.Varchar + }; + } + + public static NpgsqlParameter DeletionTrackingDisabled(bool deletionTrackingDisabled) + { + return new NpgsqlParameter + { + TypedValue = deletionTrackingDisabled, + NpgsqlDbType = NpgsqlDbType.Boolean + }; + } + + public static NpgsqlParameter Empty() + { + return new NpgsqlParameter + { + TypedValue = DBNull.Value + }; + } + } +} \ No newline at end of file diff --git a/src/SqlStreamStore.Postgres/PgSqlScriptsV2/Read.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/Read.sql new file mode 100644 index 000000000..3dee4eb07 --- /dev/null +++ b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/Read.sql @@ -0,0 +1,53 @@ +CREATE OR REPLACE FUNCTION __schema__.read( + _stream_id CHAR(42), + _count INT, + _version INT, + _forwards BOOLEAN, + _prefetch BOOLEAN +) + RETURNS SETOF REFCURSOR +AS $F$ +DECLARE + _stream_id_internal INT; + _stream_info REFCURSOR := 'stream_info'; + _messages REFCURSOR := 'messages'; +BEGIN + SELECT __schema__.streams.id_internal + INTO _stream_id_internal + FROM __schema__.streams + WHERE __schema__.streams.id = _stream_id; + + OPEN _stream_info FOR + SELECT __schema__.streams.version as stream_version, + __schema__.streams.position as position, + __schema__.streams.max_age as max_age + FROM __schema__.streams + WHERE __schema__.streams.id_internal = _stream_id_internal; + + RETURN NEXT _stream_info; + + OPEN _messages FOR + SELECT __schema__.streams.id_original AS stream_id, + __schema__.messages.message_id, + __schema__.messages.stream_version, + __schema__.messages.position, + __schema__.messages.created_utc, + __schema__.messages.type, + __schema__.messages.json_metadata, + (CASE _prefetch + WHEN TRUE THEN __schema__.messages.json_data + ELSE NULL END) + FROM __schema__.messages + INNER JOIN __schema__.streams ON __schema__.messages.stream_id_internal = __schema__.streams.id_internal + WHERE (CASE + WHEN _forwards THEN __schema__.messages.stream_version >= _version AND id_internal = _stream_id_internal + ELSE __schema__.messages.stream_version <= _version AND id_internal = _stream_id_internal END) + ORDER BY (CASE + WHEN _forwards THEN __schema__.messages.stream_version + ELSE -__schema__.messages.stream_version END) + LIMIT _count; + + RETURN NEXT _messages; +END; +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/ReadAll.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadAll.sql similarity index 72% rename from src/SqlStreamStore.Postgres/PgSqlScripts/ReadAll.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadAll.sql index ea3ad5ffa..905709e65 100644 --- a/src/SqlStreamStore.Postgres/PgSqlScripts/ReadAll.sql +++ b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadAll.sql @@ -2,23 +2,20 @@ CREATE OR REPLACE FUNCTION __schema__.read_all( _count INT, _position BIGINT, _forwards BOOLEAN, - _prefetch BOOLEAN + _prefetch BOOLEAN, + _max_position BIGINT ) - RETURNS TABLE( - stream_id VARCHAR(1000), - message_id UUID, - stream_version INT, - "position" BIGINT, - create_utc TIMESTAMP, - "type" VARCHAR(128), - json_metadata JSONB, - json_data JSONB, - max_age INT - ) + RETURNS SETOF REFCURSOR + AS $F$ + +DECLARE + _messages REFCURSOR := 'messages'; + _txinfo REFCURSOR := 'tx_info'; + BEGIN - RETURN QUERY + OPEN _messages FOR WITH messages AS ( SELECT __schema__.streams.id_original, __schema__.messages.message_id, @@ -34,6 +31,7 @@ BEGIN FROM __schema__.messages INNER JOIN __schema__.streams ON __schema__.messages.stream_id_internal = __schema__.streams.id_internal WHERE (CASE + WHEN _forwards AND _max_position > 0 THEN __schema__.messages.position >= _position AND __schema__.messages.position <= _max_position WHEN _forwards THEN __schema__.messages.position >= _position ELSE __schema__.messages.position <= _position END) ORDER BY @@ -42,6 +40,11 @@ BEGIN LIMIT _count ) SELECT * FROM messages LIMIT _count; + RETURN NEXT _messages; + + OPEN _txinfo FOR + SELECT txid_current_snapshot()::TEXT; + RETURN NEXT _txinfo; END; $F$ LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadHeadPosition.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadHeadPosition.sql new file mode 100644 index 000000000..5eddb47c8 --- /dev/null +++ b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadHeadPosition.sql @@ -0,0 +1,8 @@ +CREATE OR REPLACE FUNCTION __schema__.read_head_position() + RETURNS BIGINT +AS $F$ +BEGIN + RETURN (SELECT max(__schema__.messages.position) FROM __schema__.messages); +END; +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadJsonData.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadJsonData.sql new file mode 100644 index 000000000..1b9e3185e --- /dev/null +++ b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadJsonData.sql @@ -0,0 +1,16 @@ +CREATE OR REPLACE FUNCTION __schema__.read_json_data( + _stream_id CHAR(42), + _stream_version INT +) + RETURNS VARCHAR +AS $F$ +BEGIN + RETURN (SELECT __schema__.messages.json_data + FROM __schema__.messages + JOIN __schema__.streams ON __schema__.messages.stream_id_internal = __schema__.streams.id_internal + WHERE __schema__.messages.stream_version = _stream_version + AND __schema__.streams.id = _stream_id + LIMIT 1); +END; +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadSchemaVersion.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadSchemaVersion.sql new file mode 100644 index 000000000..dba135e67 --- /dev/null +++ b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadSchemaVersion.sql @@ -0,0 +1,13 @@ +CREATE OR REPLACE FUNCTION __schema__.read_schema_version() + RETURNS INT +AS $F$ +BEGIN + + RETURN ( + SELECT obj_description :: JSON -> 'version' + FROM obj_description('__schema__' :: REGNAMESPACE, 'pg_namespace') + ); + +END; +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadStreamHeadPosition.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadStreamHeadPosition.sql new file mode 100644 index 000000000..cefe0d638 --- /dev/null +++ b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadStreamHeadPosition.sql @@ -0,0 +1,10 @@ +CREATE OR REPLACE FUNCTION __schema__.read_stream_head_position( + _stream_id CHAR(42) +) + RETURNS BIGINT +AS $F$ +BEGIN + RETURN (SELECT (__schema__.streams.position) FROM __schema__.streams WHERE __schema__.streams.id = _stream_id); +END; +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadStreamHeadVersion.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadStreamHeadVersion.sql new file mode 100644 index 000000000..56c82a806 --- /dev/null +++ b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadStreamHeadVersion.sql @@ -0,0 +1,10 @@ +CREATE OR REPLACE FUNCTION __schema__.read_stream_head_version( + _stream_id CHAR(42) +) + RETURNS INT +AS $F$ +BEGIN + RETURN (SELECT (__schema__.streams.version) FROM __schema__.streams WHERE __schema__.streams.id = _stream_id); +END; +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadStreamVersionOfMessageId.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadStreamVersionOfMessageId.sql new file mode 100644 index 000000000..6921015dd --- /dev/null +++ b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadStreamVersionOfMessageId.sql @@ -0,0 +1,14 @@ +CREATE OR REPLACE FUNCTION __schema__.read_stream_version_of_message_id( + _stream_id_internal INT, + _message_id UUID +) + RETURNS INT +AS $F$ +BEGIN + RETURN (SELECT __schema__.messages.stream_version + FROM __schema__.messages + WHERE __schema__.messages.message_id = _message_id + AND __schema__.messages.stream_id_internal = _stream_id_internal); +END; +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.Postgres/PgSqlScriptsV2/Scavenge.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/Scavenge.sql new file mode 100644 index 000000000..686c65b0d --- /dev/null +++ b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/Scavenge.sql @@ -0,0 +1,33 @@ +CREATE OR REPLACE FUNCTION __schema__.scavenge( + _stream_id CHAR(42) +) + RETURNS TABLE( + message_id UUID + ) AS $F$ +DECLARE + _stream_id_internal INT; + _max_count INT; +BEGIN + SELECT __schema__.streams.id_internal, __schema__.streams.max_count + INTO _stream_id_internal, _max_count + FROM __schema__.streams + WHERE __schema__.streams.id = _stream_id; + + IF (_max_count IS NULL) + THEN RETURN; + END IF; + + RETURN QUERY + SELECT __schema__.messages.message_id + FROM __schema__.messages + WHERE __schema__.messages.stream_id_internal = _stream_id_internal + AND __schema__.messages.message_id NOT IN(SELECT __schema__.messages.message_id + FROM __schema__.messages + WHERE __schema__.messages.stream_id_internal = _stream_id_internal + ORDER BY __schema__.messages.stream_version desc + LIMIT _max_count) + ORDER BY __schema__.messages.stream_version; + +END; +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/Schema.cs b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/Schema.cs similarity index 94% rename from src/SqlStreamStore.Postgres/PgSqlScripts/Schema.cs rename to src/SqlStreamStore.Postgres/PgSqlScriptsV2/Schema.cs index deb1e09ab..0588c55ae 100644 --- a/src/SqlStreamStore.Postgres/PgSqlScripts/Schema.cs +++ b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/Schema.cs @@ -1,4 +1,4 @@ -namespace SqlStreamStore.PgSqlScripts +namespace SqlStreamStore.PgSqlScriptsV2 { internal class Schema { @@ -6,6 +6,7 @@ internal class Schema private readonly Scripts _scripts; public string Definition => _scripts.CreateSchema; + public string Migration => _scripts.Migration; public string DropAll => _scripts.DropAll; public string EnableExplainAnalyze => _scripts.EnableExplainAnalyze; diff --git a/src/SqlStreamStore.Postgres/PgSqlScriptsV2/Scripts.cs b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/Scripts.cs new file mode 100644 index 000000000..0ca644d41 --- /dev/null +++ b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/Scripts.cs @@ -0,0 +1,100 @@ +namespace SqlStreamStore.PgSqlScriptsV2 +{ + using System; + using System.Collections.Concurrent; + using System.IO; + using System.Reflection; + + internal class Scripts + { + private static readonly Assembly s_assembly = typeof(Scripts) + .GetTypeInfo() + .Assembly; + + private readonly string _schema; + + private readonly ConcurrentDictionary _scripts + = new ConcurrentDictionary(); + + public Scripts(string schema) + { + _schema = schema; + } + + public string DropAll => GetScript(nameof(DropAll)); + + public string EnableExplainAnalyze => GetScript(nameof(EnableExplainAnalyze)); + + private string Tables => GetScript(nameof(Tables)); + + private string AppendToStream => GetScript(nameof(AppendToStream)); + private string DeleteStream => GetScript(nameof(DeleteStream)); + + private string DeleteStreamMessages => GetScript(nameof(DeleteStreamMessages)); + private string EnforceIdempotentAppend => GetScript(nameof(EnforceIdempotentAppend)); + private string ListStreams => GetScript(nameof(ListStreams)); + private string ListStreamsStartingWith => GetScript(nameof(ListStreamsStartingWith)); + private string ListStreamsEndingWith => GetScript(nameof(ListStreamsEndingWith)); + private string ReadAll => GetScript(nameof(ReadAll)); + + private string Read => GetScript(nameof(Read)); + + private string ReadJsonData => GetScript(nameof(ReadJsonData)); + + private string ReadHeadPosition => GetScript(nameof(ReadHeadPosition)); + + private string ReadStreamHeadPosition => GetScript(nameof(ReadStreamHeadPosition)); + + private string ReadStreamHeadVersion => GetScript(nameof(ReadStreamHeadVersion)); + + private string ReadSchemaVersion => GetScript(nameof(ReadSchemaVersion)); + + private string ReadStreamVersionOfMessageId => GetScript(nameof(ReadStreamVersionOfMessageId)); + + private string Scavenge => GetScript(nameof(Scavenge)); + + private string SetStreamMetadata => GetScript(nameof(SetStreamMetadata)); + + public string Migration => GetScript(nameof(Migration)); + + public string CreateSchema => string.Join( + Environment.NewLine, + Tables, + AppendToStream, + DeleteStream, + DeleteStreamMessages, + EnforceIdempotentAppend, + ListStreams, + ListStreamsStartingWith, + ListStreamsEndingWith, + Read, + ReadAll, + ReadJsonData, + ReadHeadPosition, + ReadStreamHeadPosition, + ReadStreamHeadVersion, + ReadSchemaVersion, + ReadStreamVersionOfMessageId, + Scavenge, + SetStreamMetadata); + + private string GetScript(string name) => _scripts.GetOrAdd(name, + key => + { + using(var stream = s_assembly.GetManifestResourceStream(typeof(Scripts), $"{key}.sql")) + { + if(stream == null) + { + throw new Exception($"Embedded resource, {name}, not found. BUG!"); + } + + using(StreamReader reader = new StreamReader(stream)) + { + return reader + .ReadToEnd() + .Replace("__schema__", _schema); + } + } + }); + } +} \ No newline at end of file diff --git a/src/SqlStreamStore.Postgres/PgSqlScriptsV2/SetStreamMetadata.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/SetStreamMetadata.sql new file mode 100644 index 000000000..fe1417baf --- /dev/null +++ b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/SetStreamMetadata.sql @@ -0,0 +1,38 @@ +CREATE OR REPLACE FUNCTION __schema__.set_stream_metadata( + _stream_id CHAR(42), + _metadata_stream_id CHAR(42), + _metadata_stream_id_original CHAR(42), + _max_age INT, + _max_count INT, + _expected_version INT, + _created_utc TIMESTAMP WITH TIME ZONE, + _metadata_message __schema__.new_stream_message) + RETURNS INT AS $F$ +DECLARE + _current_version INT; +BEGIN + IF _created_utc IS NULL + THEN + _created_utc = now() at time zone 'utc'; + END IF; + + SELECT current_version + FROM __schema__.append_to_stream( + _metadata_stream_id, + _metadata_stream_id_original, + NULL, + _expected_version, + _created_utc, + ARRAY [_metadata_message] + ) + INTO _current_version; + + UPDATE __schema__.streams + SET max_age = _max_age, + max_count = _max_count + WHERE __schema__.streams.id = _stream_id; + + RETURN _current_version; +END; +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/Tables.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/Tables.sql similarity index 96% rename from src/SqlStreamStore.Postgres/PgSqlScripts/Tables.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV2/Tables.sql index 041df2ea8..e8605a905 100644 --- a/src/SqlStreamStore.Postgres/PgSqlScripts/Tables.sql +++ b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/Tables.sql @@ -21,7 +21,7 @@ CREATE INDEX IF NOT EXISTS ix_id_original_reversed ON __schema__.streams (REVERSE(id_original)); COMMENT ON SCHEMA __schema__ -IS '{ "version": 1 }'; +IS '{ "version": 2 }'; ALTER SEQUENCE __schema__.streams_seq OWNED BY __schema__.streams.id_internal; @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS __schema__.messages ( stream_version INT NOT NULL, "position" BIGINT NOT NULL DEFAULT nextval('__schema__.messages_seq'), message_id UUID NOT NULL, - created_utc TIMESTAMP NOT NULL, + created_utc TIMESTAMP WITH TIME ZONE NOT NULL, type VARCHAR(128) NOT NULL, json_data JSONB NOT NULL, json_metadata JSONB, diff --git a/src/SqlStreamStore.Postgres/PostgresStreamStore.Append.cs b/src/SqlStreamStore.Postgres/PostgresStreamStore.Append.cs index b8cc7fe49..dfd9493de 100644 --- a/src/SqlStreamStore.Postgres/PostgresStreamStore.Append.cs +++ b/src/SqlStreamStore.Postgres/PostgresStreamStore.Append.cs @@ -5,7 +5,7 @@ using System.Threading; using System.Threading.Tasks; using Npgsql; - using SqlStreamStore.PgSqlScripts; + using SqlStreamStore.PgSqlScriptsV2; using SqlStreamStore.Streams; partial class PostgresStreamStore diff --git a/src/SqlStreamStore.Postgres/PostgresStreamStore.Delete.cs b/src/SqlStreamStore.Postgres/PostgresStreamStore.Delete.cs index 0f4026f0f..ef139572a 100644 --- a/src/SqlStreamStore.Postgres/PostgresStreamStore.Delete.cs +++ b/src/SqlStreamStore.Postgres/PostgresStreamStore.Delete.cs @@ -4,7 +4,7 @@ using System.Threading; using System.Threading.Tasks; using Npgsql; - using SqlStreamStore.PgSqlScripts; + using SqlStreamStore.PgSqlScriptsV2; using SqlStreamStore.Streams; public partial class PostgresStreamStore diff --git a/src/SqlStreamStore.Postgres/PostgresStreamStore.ListStreams.cs b/src/SqlStreamStore.Postgres/PostgresStreamStore.ListStreams.cs index 9febddd6d..daebd480c 100644 --- a/src/SqlStreamStore.Postgres/PostgresStreamStore.ListStreams.cs +++ b/src/SqlStreamStore.Postgres/PostgresStreamStore.ListStreams.cs @@ -4,7 +4,7 @@ namespace SqlStreamStore using System.Threading; using System.Threading.Tasks; using Npgsql; - using SqlStreamStore.PgSqlScripts; + using SqlStreamStore.PgSqlScriptsV2; using SqlStreamStore.Streams; public partial class PostgresStreamStore diff --git a/src/SqlStreamStore.Postgres/PostgresStreamStore.Metadata.cs b/src/SqlStreamStore.Postgres/PostgresStreamStore.Metadata.cs index 1c276119f..038c4d55e 100644 --- a/src/SqlStreamStore.Postgres/PostgresStreamStore.Metadata.cs +++ b/src/SqlStreamStore.Postgres/PostgresStreamStore.Metadata.cs @@ -3,7 +3,7 @@ using System.Threading; using System.Threading.Tasks; using Npgsql; - using SqlStreamStore.PgSqlScripts; + using SqlStreamStore.PgSqlScriptsV2; using SqlStreamStore.Streams; partial class PostgresStreamStore diff --git a/src/SqlStreamStore.Postgres/PostgresStreamStore.Read.cs b/src/SqlStreamStore.Postgres/PostgresStreamStore.Read.cs index 47e175271..bc4d24dfc 100644 --- a/src/SqlStreamStore.Postgres/PostgresStreamStore.Read.cs +++ b/src/SqlStreamStore.Postgres/PostgresStreamStore.Read.cs @@ -9,7 +9,7 @@ using System.Threading; using System.Threading.Tasks; using Npgsql; - using SqlStreamStore.PgSqlScripts; + using SqlStreamStore.PgSqlScriptsV2; using SqlStreamStore.Streams; partial class PostgresStreamStore diff --git a/src/SqlStreamStore.Postgres/PostgresStreamStore.ReadAll.cs b/src/SqlStreamStore.Postgres/PostgresStreamStore.ReadAll.cs index 1eeedf0b5..af758d832 100644 --- a/src/SqlStreamStore.Postgres/PostgresStreamStore.ReadAll.cs +++ b/src/SqlStreamStore.Postgres/PostgresStreamStore.ReadAll.cs @@ -4,153 +4,227 @@ using System.Collections.Generic; using System.Data; using System.Data.Common; + using System.Linq; + using System.Text; using System.Threading; using System.Threading.Tasks; - using SqlStreamStore.PgSqlScripts; + + using Npgsql; + + using SqlStreamStore.PgSqlScriptsV2; using SqlStreamStore.Streams; + public class PostgresReadAllPage : ReadAllPage + { + public List TransactionIds { get; } + public PostgresReadAllPage( + long fromPosition, + long nextPosition, + bool isEnd, + ReadDirection direction, + StreamMessage[] messages = null, + string transactionSnapshot = null) : base(fromPosition, nextPosition, isEnd, direction, messages) + { + TransactionIds = ParseTransactionSnapshot(transactionSnapshot); + } + + private static List ParseTransactionSnapshot(string transactionSnapshot) + { + if (!string.IsNullOrWhiteSpace(transactionSnapshot)) + { + var splitResult = transactionSnapshot.Split(':'); + + if(splitResult.Length > 2 && !string.IsNullOrWhiteSpace(splitResult[2])) + { + return splitResult[2].Split(',').Select(x => Convert.ToInt64(x)).ToList(); + } + } + + return new List(); + } + } + partial class PostgresStreamStore { - protected override async Task ReadAllForwardsInternal( + protected override async Task ReadAllForwardsInternal( long fromPositionExclusive, int maxCount, bool prefetch, - ReadNextAllPage readNext, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + long fromMaxPositionInclusive = -1) { maxCount = maxCount == int.MaxValue ? maxCount - 1 : maxCount; - using(var connection = await OpenConnection(cancellationToken)) - using(var transaction = connection.BeginTransaction()) - using(var command = BuildFunctionCommand( + var refcursorSql = new StringBuilder(); + + using (var connection = await OpenConnection(cancellationToken)) + using (var transaction = connection.BeginTransaction()) + { + using (var command = BuildFunctionCommand( _schema.ReadAll, transaction, Parameters.Count(maxCount + 1), Parameters.Position(fromPositionExclusive), Parameters.ReadDirection(ReadDirection.Forward), - Parameters.Prefetch(prefetch))) - using(var reader = await command - .ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken) - .ConfigureAwait(false)) - { - if(!reader.HasRows) + Parameters.Prefetch(prefetch), + Parameters.Position(fromMaxPositionInclusive))) + using (var reader = await command + .ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken) + .ConfigureAwait(false)) { - return new ReadAllPage( - fromPositionExclusive, - fromPositionExclusive, - true, - ReadDirection.Forward, - readNext, - Array.Empty()); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + refcursorSql.AppendLine(Schema.FetchAll(reader.GetString(0))); + } } - var messages = new List<(StreamMessage message, int? maxAge)>(); - - while(await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + using (var command = new NpgsqlCommand(refcursorSql.ToString(), transaction.Connection, transaction)) + using (var reader = await command + .ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken) + .ConfigureAwait(false)) { - if(messages.Count == maxCount) + + if (!reader.HasRows) { - messages.Add(default); + return new PostgresReadAllPage( + fromPositionExclusive, + fromPositionExclusive, + true, + ReadDirection.Forward, + Array.Empty()); } - else + + var messages = new List<(StreamMessage message, int? maxAge)>(); + + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) { - var streamIdInfo = new StreamIdInfo(reader.GetString(0)); - var (message, maxAge, _) = - await ReadAllStreamMessage(reader, streamIdInfo.PostgresqlStreamId, prefetch); - messages.Add((message, maxAge)); + if (messages.Count == maxCount) + { + messages.Add(default); + } + else + { + var streamIdInfo = new StreamIdInfo(reader.GetString(0)); + var (message, maxAge, _) = + await ReadAllStreamMessage(reader, streamIdInfo.PostgresqlStreamId, prefetch); + messages.Add((message, maxAge)); + } + } + + string transactionSnapshot = string.Empty; + await reader.NextResultAsync(cancellationToken).ConfigureAwait(false); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + transactionSnapshot = await reader.GetFieldValueAsync(0, cancellationToken); } - } - bool isEnd = true; + bool isEnd = true; - if(messages.Count == maxCount + 1) // An extra row was read, we're not at the end - { - isEnd = false; - messages.RemoveAt(maxCount); - } + if (messages.Count == maxCount + 1) // An extra row was read, we're not at the end + { + isEnd = false; + messages.RemoveAt(maxCount); + } - var filteredMessages = FilterExpired(messages); + var filteredMessages = FilterExpired(messages); - var nextPosition = filteredMessages[filteredMessages.Count - 1].Position + 1; + var nextPosition = filteredMessages[filteredMessages.Count - 1].Position + 1; - return new ReadAllPage( - fromPositionExclusive, - nextPosition, - isEnd, - ReadDirection.Forward, - readNext, - filteredMessages.ToArray()); + return new PostgresReadAllPage( + fromPositionExclusive, + nextPosition, + isEnd, + ReadDirection.Forward, + filteredMessages.ToArray(), + transactionSnapshot); + } } } - protected override async Task ReadAllBackwardsInternal( + protected override async Task ReadAllBackwardsInternal( long fromPositionExclusive, int maxCount, bool prefetch, - ReadNextAllPage readNext, CancellationToken cancellationToken) { maxCount = maxCount == int.MaxValue ? maxCount - 1 : maxCount; var ordinal = fromPositionExclusive == Position.End ? long.MaxValue : fromPositionExclusive; - using(var connection = await OpenConnection(cancellationToken)) - using(var transaction = connection.BeginTransaction()) - using(var command = BuildFunctionCommand( - _schema.ReadAll, - transaction, - Parameters.Count(maxCount + 1), - Parameters.Position(ordinal), - Parameters.ReadDirection(ReadDirection.Backward), - Parameters.Prefetch(prefetch))) - using(var reader = await command - .ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken) - .ConfigureAwait(false)) + var refcursorSql = new StringBuilder(); + + using (var connection = await OpenConnection(cancellationToken)) + using (var transaction = connection.BeginTransaction()) { - if(!reader.HasRows) + using (var command = BuildFunctionCommand( + _schema.ReadAll, + transaction, + Parameters.Count(maxCount + 1), + Parameters.Position(ordinal), + Parameters.ReadDirection(ReadDirection.Backward), + Parameters.Prefetch(prefetch), + Parameters.Position(-1))) { - // When reading backwards and there are no more items, then next position is LongPosition.Start, - // regardless of what the fromPosition is. - return new ReadAllPage( - Position.Start, - Position.Start, - true, - ReadDirection.Backward, - readNext, - Array.Empty()); - } + using (var reader = await command + .ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken) + .ConfigureAwait(false)) - var messages = new List<(StreamMessage message, int? maxAge)>(); + { + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + refcursorSql.AppendLine(Schema.FetchAll(reader.GetString(0))); + } + } + } - long lastOrdinal = 0; - while(await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + using (var command = new NpgsqlCommand(refcursorSql.ToString(), transaction.Connection, transaction)) + using (var reader = await command + .ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken) + .ConfigureAwait(false)) { - var streamIdInfo = new StreamIdInfo(reader.GetString(0)); - var (message, maxAge, position) = await ReadAllStreamMessage(reader, streamIdInfo.PostgresqlStreamId, prefetch); - messages.Add((message, maxAge)); + if (!reader.HasRows) + { + // When reading backwards and there are no more items, then next position is LongPosition.Start, + // regardless of what the fromPosition is. + return new PostgresReadAllPage( + Position.Start, + Position.Start, + true, + ReadDirection.Backward, + Array.Empty()); + } - lastOrdinal = position; - } + var messages = new List<(StreamMessage message, int? maxAge)>(); - bool isEnd = true; - var nextPosition = lastOrdinal; + long lastOrdinal = 0; + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + var streamIdInfo = new StreamIdInfo(reader.GetString(0)); + var (message, maxAge, position) = await ReadAllStreamMessage(reader, streamIdInfo.PostgresqlStreamId, prefetch); + messages.Add((message, maxAge)); - if(messages.Count == maxCount + 1) // An extra row was read, we're not at the end - { - isEnd = false; - messages.RemoveAt(maxCount); - } + lastOrdinal = position; + } + + bool isEnd = true; + var nextPosition = lastOrdinal; + + if (messages.Count == maxCount + 1) // An extra row was read, we're not at the end + { + isEnd = false; + messages.RemoveAt(maxCount); + } - var filteredMessages = FilterExpired(messages); + var filteredMessages = FilterExpired(messages); - fromPositionExclusive = filteredMessages.Count > 0 ? filteredMessages[0].Position : 0; + fromPositionExclusive = filteredMessages.Count > 0 ? filteredMessages[0].Position : 0; - return new ReadAllPage( - fromPositionExclusive, - nextPosition, - isEnd, - ReadDirection.Backward, - readNext, - filteredMessages.ToArray()); + return new PostgresReadAllPage( + fromPositionExclusive, + nextPosition, + isEnd, + ReadDirection.Backward, + filteredMessages.ToArray()); + } } } @@ -161,12 +235,12 @@ protected override async Task ReadAllBackwardsInternal( { async Task ReadString(int ordinal) { - if(reader.IsDBNull(ordinal)) + if (reader.IsDBNull(ordinal)) { return null; } - using(var textReader = reader.GetTextReader(ordinal)) + using (var textReader = reader.GetTextReader(ordinal)) { return await textReader.ReadToEndAsync().ConfigureAwait(false); } @@ -179,7 +253,7 @@ async Task ReadString(int ordinal) var type = reader.GetString(5); var jsonMetadata = await ReadString(6); - if(prefetch) + if (prefetch) { return ( new StreamMessage( diff --git a/src/SqlStreamStore.Postgres/PostgresStreamStore.Subscriptions.cs b/src/SqlStreamStore.Postgres/PostgresStreamStore.Subscriptions.cs index 24440fef5..e1d98bb1d 100644 --- a/src/SqlStreamStore.Postgres/PostgresStreamStore.Subscriptions.cs +++ b/src/SqlStreamStore.Postgres/PostgresStreamStore.Subscriptions.cs @@ -32,7 +32,7 @@ protected override IAllStreamSubscription SubscribeToAllInternal( HasCaughtUp hasCaughtUp, bool prefetchJsonData, string name) - => new AllStreamSubscription( + => new AllStreamSubscription( fromPosition, this, GetStoreObservable, diff --git a/src/SqlStreamStore.Postgres/PostgresStreamStore.cs b/src/SqlStreamStore.Postgres/PostgresStreamStore.cs index 32da063ed..bb52e24c3 100644 --- a/src/SqlStreamStore.Postgres/PostgresStreamStore.cs +++ b/src/SqlStreamStore.Postgres/PostgresStreamStore.cs @@ -3,25 +3,26 @@ using System; using System.Collections.Generic; using System.Data; + using System.Linq; using System.Threading; using System.Threading.Tasks; using Npgsql; using SqlStreamStore.Infrastructure; using SqlStreamStore.Logging; - using SqlStreamStore.PgSqlScripts; + using SqlStreamStore.PgSqlScriptsV2; using SqlStreamStore.Subscriptions; /// /// Represents a PostgreSQL stream store implementation. /// - public partial class PostgresStreamStore : StreamStoreBase + public partial class PostgresStreamStore : StreamStoreBase { private readonly PostgresStreamStoreSettings _settings; private readonly Func _createConnection; private readonly Schema _schema; private readonly Lazy _streamStoreNotifier; - public const int CurrentVersion = 1; + public const int CurrentVersion = 2; /// /// Initializes a new instance of @@ -259,5 +260,70 @@ public string GetSchemaCreationScript() { return _schema.Definition; } + + /// + /// Returns the script that can be used to migrate to the latest schema version 2. + /// + /// The database creation script. + public string GetMigrationScript() + { + return _schema.Migration; + } + + //protected override async Task HandleGap( + // PostgresReadAllPage page, + // long fromPositionInclusive, + // int maxCount, + // bool prefetchJsonData, + // CancellationToken cancellationToken) + //{ + // if (page.Messages.Length == 0 || DateTime.UtcNow - page.Messages[page.Messages.Length - 1].CreatedUtc > TimeSpan.FromMinutes(5)) + // return page; + + // // TODO: FIXIT + // // Check for gap between last page and this. + // //if (page.Messages[0].Position != fromPositionInclusive) + // //{ + // // Logger.InfoFormat("Gap detected at lower page boundary. Potentially could have lost {lostMessageCount} events if the gap is transient", page.Messages[0].Position - fromPositionInclusive); + // // page = await HandleGap(page, fromPositionInclusive, maxCount, prefetchJsonData, cancellationToken); + // // //if (!page.IsEnd || page.Messages.Length == 1) + // // // Logger.DebugFormat("Gap detected at lower page boundary. Potentially could have lost {lostMessageCount} events if the gap is transient", page.Messages[0].Position - fromPositionInclusive); + // // //page = await ReloadAfterDelay(fromPositionInclusive, maxCount, prefetchJsonData, ReadNext, cancellationToken); + // //} + + // // check for gap in messages collection + // for (int i = 0; i < page.Messages.Length - 1; i++) + // { + // var expectedNextPosition = page.Messages[i].Position + 1; + // if (expectedNextPosition != page.Messages[i + 1].Position) + // { + // Logger.InfoFormat("Gap detected in " + (page.IsEnd ? "last" : "(NOT the last)") + " page. Returning partial page {fromPosition}-{toPosition}", fromPositionInclusive, fromPositionInclusive + i + 1); + + // PostgresReadAllPage requeryPage; + // var maxPosition = page.Messages[page.Messages.Length - 1].Position; + // do + // { + // requeryPage = await ReadAllForwardsInternal(fromPositionInclusive, maxCount, prefetchJsonData, cancellationToken, maxPosition); + // } while (page.TransactionIds.Intersect(requeryPage.TransactionIds).Any()); + + // return requeryPage; + + // // switched this to return the partial page, then re-issue load starting at gap + // // this speeds up the retry instead of taking a 3 second delay immediately + // //var messagesBeforeGap = new StreamMessage[i+1]; + // //page.Messages.Take(i+1).ToArray().CopyTo(messagesBeforeGap, 0); + // //return new ReadAllPage(page.FromPosition, maxPosition, page.IsEnd, page.Direction, ReadNext, messagesBeforeGap); + // } + // } + + // //ReadAllPage requeryPage; + // //var maxPosition = pageWithGap.Messages[pageWithGap.Messages.Length - 1].Position; + // //do + // //{ + // // requeryPage = await ReadAllForwardsInternal(fromPositionInclusive, maxCount, prefetchJsonData, readNext, cancellationToken, maxPosition); + // //} while (pageWithGap.TxSnapshot.CurrentTxIds.Intersect(requeryPage.TxSnapshot.CurrentTxIds).Any()); + + // return page; + //} } } diff --git a/src/SqlStreamStore.Postgres/Properties/AssemblyInfo.cs b/src/SqlStreamStore.Postgres/Properties/AssemblyInfo.cs index 8ddfbb782..96229f8c5 100644 --- a/src/SqlStreamStore.Postgres/Properties/AssemblyInfo.cs +++ b/src/SqlStreamStore.Postgres/Properties/AssemblyInfo.cs @@ -1,3 +1,4 @@ using System.Runtime.CompilerServices; -[assembly: InternalsVisibleTo("SqlStreamStore.Postgres.Tests")] \ No newline at end of file +[assembly: InternalsVisibleTo("SqlStreamStore.Postgres.Tests")] +[assembly: InternalsVisibleTo("SqlStreamStore.PostgresV2")] diff --git a/src/SqlStreamStore.Postgres/SqlStreamStore.Postgres.csproj b/src/SqlStreamStore.Postgres/SqlStreamStore.Postgres.csproj index 9702d8626..164461878 100644 --- a/src/SqlStreamStore.Postgres/SqlStreamStore.Postgres.csproj +++ b/src/SqlStreamStore.Postgres/SqlStreamStore.Postgres.csproj @@ -2,17 +2,16 @@ Postgre SQL provider for SQL StreamStore Stream Store - Postgres - netstandard2.0 + netstandard2.0 SqlStreamStore.Postgres SqlStreamStore.Postgres SqlStreamStore - 1.6.0 - + - + diff --git a/src/SqlStreamStore.SchemaCreationScriptTool/SqlStreamStore.SchemaCreationScriptTool.csproj b/src/SqlStreamStore.SchemaCreationScriptTool/SqlStreamStore.SchemaCreationScriptTool.csproj index 3381c6f9f..d284c7bba 100644 --- a/src/SqlStreamStore.SchemaCreationScriptTool/SqlStreamStore.SchemaCreationScriptTool.csproj +++ b/src/SqlStreamStore.SchemaCreationScriptTool/SqlStreamStore.SchemaCreationScriptTool.csproj @@ -4,7 +4,7 @@ A tool to generate the schema creation script of the various SQL based implementations. SQL Stream Store - Schema Creation Script Tool Exe - netcoreapp2.1 + net6.0 true SqlStreamStore.SchemaCreationScriptTool SqlStreamStore.SchemaCreationScriptTool @@ -23,9 +23,9 @@ - - - - + + + + \ No newline at end of file diff --git a/src/SqlStreamStore.Sqlite/SqlStreamStore.Sqlite.csproj b/src/SqlStreamStore.Sqlite/SqlStreamStore.Sqlite.csproj index f25a980ec..f55d77460 100644 --- a/src/SqlStreamStore.Sqlite/SqlStreamStore.Sqlite.csproj +++ b/src/SqlStreamStore.Sqlite/SqlStreamStore.Sqlite.csproj @@ -2,7 +2,7 @@ Sqlite provider for SQL Stream Store SQL Stream Store - Sqlite Provider - netstandard2.0 + netstandard2.0 true SqlStreamStore.Sqlite SqlStreamStore.Sqlite @@ -15,8 +15,8 @@ - - + + diff --git a/src/SqlStreamStore.Sqlite/SqliteStreamStore.ReadAll.cs b/src/SqlStreamStore.Sqlite/SqliteStreamStore.ReadAll.cs index 97e4a893d..7dbbd1b25 100644 --- a/src/SqlStreamStore.Sqlite/SqliteStreamStore.ReadAll.cs +++ b/src/SqlStreamStore.Sqlite/SqliteStreamStore.ReadAll.cs @@ -12,8 +12,8 @@ protected override async Task ReadAllForwardsInternal( long fromPositionExclusive, int maxCount, bool prefetch, - ReadNextAllPage readNext, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + long fromMaxPositionInclusive = -1) { GuardAgainstDisposed(); cancellationToken.ThrowIfCancellationRequested(); @@ -29,8 +29,7 @@ protected override async Task ReadAllForwardsInternal( Position.Start, Position.Start, true, - ReadDirection.Forward, - readNext); + ReadDirection.Forward); } if(allStreamPosition < fromPositionExclusive) @@ -39,8 +38,7 @@ protected override async Task ReadAllForwardsInternal( fromPositionExclusive, fromPositionExclusive, true, - ReadDirection.Forward, - readNext); + ReadDirection.Forward); } var remaining = await connection.AllStream() @@ -52,8 +50,7 @@ protected override async Task ReadAllForwardsInternal( fromPositionExclusive, Position.End, true, - ReadDirection.Forward, - readNext); + ReadDirection.Forward); } var messages = await connection.AllStream() @@ -74,7 +71,6 @@ protected override async Task ReadAllForwardsInternal( nextPosition, isEnd, ReadDirection.Forward, - readNext, messages.ToArray()); } } @@ -83,7 +79,6 @@ protected override async Task ReadAllBackwardsInternal( long fromPosition, int maxCount, bool prefetch, - ReadNextAllPage readNext, CancellationToken cancellationToken) { GuardAgainstDisposed(); @@ -100,8 +95,7 @@ protected override async Task ReadAllBackwardsInternal( Position.Start, Position.Start, true, - ReadDirection.Backward, - readNext); + ReadDirection.Backward); } if(fromPosition == Position.End) @@ -115,8 +109,7 @@ protected override async Task ReadAllBackwardsInternal( fromPosition, fromPosition, true, - ReadDirection.Backward, - readNext); + ReadDirection.Backward); } // For reading $all, in the case where no events have been entered into @@ -134,8 +127,7 @@ protected override async Task ReadAllBackwardsInternal( allStreamPosition ?? Position.Start, Position.End, true, - ReadDirection.Backward, - readNext); + ReadDirection.Backward); } var messages = await connection.AllStream() @@ -154,7 +146,6 @@ protected override async Task ReadAllBackwardsInternal( nextPosition, isEnd, ReadDirection.Backward, - readNext, messages.ToArray()); } } diff --git a/src/SqlStreamStore.Sqlite/SqliteStreamStore.Subscriptions.cs b/src/SqlStreamStore.Sqlite/SqliteStreamStore.Subscriptions.cs index cc670950d..c43bd05a6 100644 --- a/src/SqlStreamStore.Sqlite/SqliteStreamStore.Subscriptions.cs +++ b/src/SqlStreamStore.Sqlite/SqliteStreamStore.Subscriptions.cs @@ -2,6 +2,7 @@ namespace SqlStreamStore { using System; using SqlStreamStore.Infrastructure; + using SqlStreamStore.Streams; using SqlStreamStore.Subscriptions; public partial class SqliteStreamStore @@ -37,7 +38,7 @@ protected override IAllStreamSubscription SubscribeToAllInternal( bool prefetchJsonData, string name) { - return new AllStreamSubscription( + return new AllStreamSubscription( fromPosition, this, GetStoreObservable, diff --git a/src/SqlStreamStore.Sqlite/SqliteStreamStore.cs b/src/SqlStreamStore.Sqlite/SqliteStreamStore.cs index 7d53c5e2e..e843d1f7b 100644 --- a/src/SqlStreamStore.Sqlite/SqliteStreamStore.cs +++ b/src/SqlStreamStore.Sqlite/SqliteStreamStore.cs @@ -15,7 +15,7 @@ namespace SqlStreamStore using SqlStreamStore.Streams; using SqlStreamStore.Subscriptions; - public partial class SqliteStreamStore : StreamStoreBase + public partial class SqliteStreamStore : StreamStoreBase { private static readonly System.Collections.Concurrent.ConcurrentDictionary _scripts = new ConcurrentDictionary(); private static readonly Assembly s_assembly = typeof(SqliteStreamStore) diff --git a/src/SqlStreamStore.TestUtils/MsSql/SqlServerContainer.cs b/src/SqlStreamStore.TestUtils/MsSql/SqlServerContainer.cs index 4eeed5339..d5796e36e 100644 --- a/src/SqlStreamStore.TestUtils/MsSql/SqlServerContainer.cs +++ b/src/SqlStreamStore.TestUtils/MsSql/SqlServerContainer.cs @@ -37,7 +37,7 @@ public SqlConnection CreateConnection() => new SqlConnection(CreateConnectionStringBuilder().ConnectionString); public SqlConnectionStringBuilder CreateConnectionStringBuilder() - => new SqlConnectionStringBuilder($"server=localhost,{HostPort};User Id=sa;Password={Password};Initial Catalog=master"); + => new SqlConnectionStringBuilder($"server=localhost,{HostPort};User Id=sa;Password={Password};Initial Catalog=master;TrustServerCertificate=true"); public async Task Start(CancellationToken cancellationToken = default) { diff --git a/src/SqlStreamStore.TestUtils/Postgres/LibLogNpgsqlLogProvider.cs b/src/SqlStreamStore.TestUtils/Postgres/LibLogNpgsqlLogProvider.cs index d0d6116d2..32311e2b0 100644 --- a/src/SqlStreamStore.TestUtils/Postgres/LibLogNpgsqlLogProvider.cs +++ b/src/SqlStreamStore.TestUtils/Postgres/LibLogNpgsqlLogProvider.cs @@ -23,7 +23,7 @@ public LibLogNpgsqlLogger(ILog logger, string name) _name = name; } - public override bool IsEnabled(NpgsqlLogLevel level) => true; + public override bool IsEnabled(NpgsqlLogLevel level) => false; public override void Log(NpgsqlLogLevel level, int connectorId, string msg, Exception exception = null) => _logger.Info($@"[{level:G}] [{_name}] (Connector Id: {connectorId}); {msg}; {FormatOptionalException(exception)}"); diff --git a/src/SqlStreamStore.TestUtils/Postgres/PostgresContainer.cs b/src/SqlStreamStore.TestUtils/Postgres/PostgresContainer.cs index 1d93955bf..08a4e78a5 100644 --- a/src/SqlStreamStore.TestUtils/Postgres/PostgresContainer.cs +++ b/src/SqlStreamStore.TestUtils/Postgres/PostgresContainer.cs @@ -1,10 +1,14 @@ namespace SqlStreamStore.TestUtils.Postgres { using System; + using System.Globalization; + using System.Linq; using System.Threading; using System.Threading.Tasks; - using Ductus.FluentDocker.Builders; + using Ductus.FluentDocker.Extensions; + using Ductus.FluentDocker.Model.Containers; using Ductus.FluentDocker.Services; + using Ductus.FluentDocker.Services.Extensions; using Npgsql; using Polly; @@ -13,31 +17,45 @@ public class PostgresContainer : PostgresDatabaseManager private readonly IContainerService _containerService; private const string Image = "postgres:10.4-alpine"; private const string ContainerName = "sql-stream-store-tests-postgres"; - private const int Port = 5432; - public override string ConnectionString => ConnectionStringBuilder.ConnectionString; - - public PostgresContainer(string databaseName) + public PostgresContainer(string schema, string databaseName, float cpu = float.MinValue) : base(databaseName) { - _containerService = new Builder() - .UseContainer() - .WithName(ContainerName) - .UseImage(Image) - .KeepRunning() - .ReuseIfExists() - .ExposePort(Port, Port) - .Command("-N", "500") - .Build(); + CultureInfo.DefaultThreadCurrentCulture = new CultureInfo("en-US"); + + var hosts = new Hosts().Discover(); + var host = hosts.FirstOrDefault(x => x.IsNative) ?? hosts.FirstOrDefault(x => x.Name == "default"); + + + _containerService = host.Create(Image, false, new ContainerCreateParams + { + Cpus = cpu, + Name = $"{schema}-{ContainerName}", + PortMappings = new[] { "5432" } + + }, false, true, command: "-N 500"); + + + //_containerService = new Builder() + // .UseContainer() + // .WithName(ContainerName) + // .UseImage(Image) + // .KeepRunning() + // .ReuseIfExists() + // .ExposePort(Port, Port) + // .Command("-N", "500") + // .WithParentCGroup() + // .Build(); } public async Task Start(CancellationToken cancellationToken = default) { _containerService.Start(); + _containerService.WaitForRunning(); await Policy .Handle() - .WaitAndRetryAsync(30, _ => TimeSpan.FromMilliseconds(500)) + .WaitAndRetryAsync(100, _ => TimeSpan.FromMilliseconds(500)) .ExecuteAsync(async () => { using(var connection = new NpgsqlConnection(DefaultConnectionString)) @@ -47,17 +65,28 @@ await Policy }); } - private NpgsqlConnectionStringBuilder ConnectionStringBuilder => new NpgsqlConnectionStringBuilder + public override string GenerateConnectionString(string applicationName = "default") + { + + return new NpgsqlConnectionStringBuilder(ConnectionStringBuilder.ConnectionString) + { + ApplicationName = applicationName, + Port = _containerService.ToHostExposedEndpoint("5432/tcp").Port + }.ConnectionString; + } + + private NpgsqlConnectionStringBuilder ConnectionStringBuilder => new() { Database = DatabaseName, Password = Environment.OSVersion.IsWindows() ? "password" : null, - Port = Port, Username = "postgres", Host = "localhost", Pooling = true, - MaxPoolSize = 1024 + MaxPoolSize = 1024, + Timeout = 300, + IncludeErrorDetails = false }; } } diff --git a/src/SqlStreamStore.TestUtils/Postgres/PostgresDatabaseManager.cs b/src/SqlStreamStore.TestUtils/Postgres/PostgresDatabaseManager.cs index 220246293..2b3db4ca2 100644 --- a/src/SqlStreamStore.TestUtils/Postgres/PostgresDatabaseManager.cs +++ b/src/SqlStreamStore.TestUtils/Postgres/PostgresDatabaseManager.cs @@ -11,18 +11,18 @@ public abstract class PostgresDatabaseManager private bool _started; - protected string DefaultConnectionString => new NpgsqlConnectionStringBuilder(ConnectionString) + protected string DefaultConnectionString => new NpgsqlConnectionStringBuilder(GenerateConnectionString()) { Database = null }.ConnectionString; - public abstract string ConnectionString { get; } + public abstract string GenerateConnectionString(string applicationName = "default"); static PostgresDatabaseManager() { #if DEBUG - NpgsqlLogManager.IsParameterLoggingEnabled = true; - NpgsqlLogManager.Provider = new LibLogNpgsqlLogProvider(); + NpgsqlLogManager.IsParameterLoggingEnabled = false; + //NpgsqlLogManager.Provider = new LibLogNpgsqlLogProvider(); #endif } diff --git a/src/SqlStreamStore.TestUtils/SqlStreamStore.TestUtils.csproj b/src/SqlStreamStore.TestUtils/SqlStreamStore.TestUtils.csproj index 4c7a7731b..13180dfe4 100644 --- a/src/SqlStreamStore.TestUtils/SqlStreamStore.TestUtils.csproj +++ b/src/SqlStreamStore.TestUtils/SqlStreamStore.TestUtils.csproj @@ -1,21 +1,24 @@  - netcoreapp2.1 + net6.0 SqlStreamStore.TestUtils false - - - - - + + + + + - - + + + + + - - + + all @@ -25,4 +28,8 @@ + + + + \ No newline at end of file diff --git a/src/SqlStreamStore/IReadonlyStreamStore.cs b/src/SqlStreamStore/IReadonlyStreamStore.cs index 3452e2e6d..a15f30889 100644 --- a/src/SqlStreamStore/IReadonlyStreamStore.cs +++ b/src/SqlStreamStore/IReadonlyStreamStore.cs @@ -3,68 +3,12 @@ using System; using System.Threading; using System.Threading.Tasks; + using MorseCode.ITask; using SqlStreamStore.Streams; using SqlStreamStore.Subscriptions; - /// - /// Represents a readonly stream store. - /// public interface IReadonlyStreamStore : IDisposable { - /// - /// Reads messages from all streams forwards. - /// - /// - /// The position to start reading from. Use to start from the beginning. - /// Note: messages that have expired will be filtered out. - /// - /// - /// The maximum number of messages to read (int.MaxValue is a bad idea). - /// - /// - /// Prefetches the message data as part of the page read. This means a single request to the server - /// but a higher payload size. - /// - /// - /// The cancellation instruction. - /// - /// - /// An presenting the result of the read. If all messages read have expired - /// then the message collection MAY be empty. - /// - Task ReadAllForwards( - long fromPositionInclusive, - int maxCount, - bool prefetchJsonData = true, - CancellationToken cancellationToken = default); - - /// - /// Reads messages from all streams backwards. - /// - /// - /// The position to start reading from. Use to start from the end. - /// Note: messages that have expired will be filtered out. - /// - /// - /// The maximum number of messages to read (int.MaxValue is a bad idea). - /// - /// - /// Prefetches the message data as part of the page read. This means a single request to the server - /// but a higher payload size. - /// - /// - /// The cancellation instruction. - /// - /// - /// An presenting the result of the read. If all messages read have expired - /// then the message collection MAY be empty. - /// - Task ReadAllBackwards( - long fromPositionInclusive, - int maxCount, - bool prefetchJsonData = true, - CancellationToken cancellationToken = default); - /// /// Reads messages from a stream forwards. /// @@ -304,4 +248,64 @@ Task ListStreams( event Action OnDispose; } + + /// + /// Represents a readonly stream store. + /// + public interface IReadonlyStreamStore : IReadonlyStreamStore where TReadAllPage : IReadAllPage + { + /// + /// Reads messages from all streams forwards. + /// + /// + /// The position to start reading from. Use to start from the beginning. + /// Note: messages that have expired will be filtered out. + /// + /// + /// The maximum number of messages to read (int.MaxValue is a bad idea). + /// + /// + /// Prefetches the message data as part of the page read. This means a single request to the server + /// but a higher payload size. + /// + /// + /// The cancellation instruction. + /// + /// + /// An presenting the result of the read. If all messages read have expired + /// then the message collection MAY be empty. + /// + ITask ReadAllForwards( + long fromPositionInclusive, + int maxCount, + bool prefetchJsonData = true, + CancellationToken cancellationToken = default); + + /// + /// Reads messages from all streams backwards. + /// + /// + /// The position to start reading from. Use to start from the end. + /// Note: messages that have expired will be filtered out. + /// + /// + /// The maximum number of messages to read (int.MaxValue is a bad idea). + /// + /// + /// Prefetches the message data as part of the page read. This means a single request to the server + /// but a higher payload size. + /// + /// + /// The cancellation instruction. + /// + /// + /// An presenting the result of the read. If all messages read have expired + /// then the message collection MAY be empty. + /// + ITask ReadAllBackwards( + long fromPositionInclusive, + int maxCount, + bool prefetchJsonData = true, + CancellationToken cancellationToken = default); + } } \ No newline at end of file diff --git a/src/SqlStreamStore/IStreamStore.cs b/src/SqlStreamStore/IStreamStore.cs index aa21cc89c..1d1d7bae6 100644 --- a/src/SqlStreamStore/IStreamStore.cs +++ b/src/SqlStreamStore/IStreamStore.cs @@ -121,4 +121,9 @@ Task SetStreamMetadata( string metadataJson = null, CancellationToken cancellationToken = default); } + + // TODO + public interface IStreamStore : IStreamStore, IReadonlyStreamStore where TReadPage : IReadAllPage + { + } } diff --git a/src/SqlStreamStore/InMemory/InMemoryStreamStore.cs b/src/SqlStreamStore/InMemory/InMemoryStreamStore.cs index aa9fd88ed..80d9f4e5e 100644 --- a/src/SqlStreamStore/InMemory/InMemoryStreamStore.cs +++ b/src/SqlStreamStore/InMemory/InMemoryStreamStore.cs @@ -17,7 +17,7 @@ namespace SqlStreamStore /// /// Represents an in-memory implementation of a stream store. Use for testing or high/speed + volatile scenarios. /// - public sealed class InMemoryStreamStore : StreamStoreBase + public sealed class InMemoryStreamStore : StreamStoreBase { private readonly InMemoryAllStream _allStream = new InMemoryAllStream(); private readonly GetUtcNow _getUtcNow; @@ -296,7 +296,7 @@ private void DeleteStream(string streamId, int expectedVersion) } protected override Task ReadAllForwardsInternal(long fromPositionExclusive, int maxCount, - bool prefetch, ReadNextAllPage readNext, CancellationToken cancellationToken) + bool prefetch, CancellationToken cancellationToken, long fromMaxPositionInclusive = -1) { GuardAgainstDisposed(); cancellationToken.ThrowIfCancellationRequested(); @@ -307,7 +307,7 @@ protected override Task ReadAllForwardsInternal(long fromPositionEx var current = _allStream.First; if(current.Next == null) //Empty store { - var result = new ReadAllPage(Position.Start, Position.Start, true, ReadDirection.Forward, readNext); + var result = new ReadAllPage(Position.Start, Position.Start, true, ReadDirection.Forward); return Task.FromResult(result); } @@ -317,7 +317,7 @@ protected override Task ReadAllForwardsInternal(long fromPositionEx if(current.Next == null) // fromPosition is past end of store { var result = new ReadAllPage(fromPositionExclusive, fromPositionExclusive, true, - ReadDirection.Forward, readNext); + ReadDirection.Forward); return Task.FromResult(result); } previous = current; @@ -372,7 +372,6 @@ protected override Task ReadAllForwardsInternal(long fromPositionEx nextPosition, isEnd, ReadDirection.Forward, - readNext, messages.ToArray()); return Task.FromResult(page); @@ -383,7 +382,6 @@ protected override Task ReadAllBackwardsInternal( long fromPositionExclusive, int maxCount, bool prefetch, - ReadNextAllPage readNext, CancellationToken cancellationToken) { GuardAgainstDisposed(); @@ -399,7 +397,7 @@ protected override Task ReadAllBackwardsInternal( var current = _allStream.First; if(current.Next == null) //Empty store { - var result = new ReadAllPage(Position.Start, Position.Start, true, ReadDirection.Backward, readNext); + var result = new ReadAllPage(Position.Start, Position.Start, true, ReadDirection.Backward); return Task.FromResult(result); } @@ -409,7 +407,7 @@ protected override Task ReadAllBackwardsInternal( if(current.Next == null) // fromPosition is past end of store { var result = new ReadAllPage(fromPositionExclusive, fromPositionExclusive, true, - ReadDirection.Backward, readNext); + ReadDirection.Backward); return Task.FromResult(result); } previous = current; @@ -476,7 +474,6 @@ protected override Task ReadAllBackwardsInternal( nextPosition, isEnd, ReadDirection.Backward, - readNext, messages.ToArray()); return Task.FromResult(page); @@ -721,7 +718,7 @@ protected override IAllStreamSubscription SubscribeToAllInternal( bool prefetchJsonData, string name) { - return new AllStreamSubscription( + return new AllStreamSubscription( fromPosition, this, _subscriptions, diff --git a/src/SqlStreamStore/Infrastructure/ReadonlyStreamStoreBase.cs b/src/SqlStreamStore/Infrastructure/ReadonlyStreamStoreBase.cs index 1b20be17e..3e0be1a87 100644 --- a/src/SqlStreamStore/Infrastructure/ReadonlyStreamStoreBase.cs +++ b/src/SqlStreamStore/Infrastructure/ReadonlyStreamStoreBase.cs @@ -4,6 +4,7 @@ namespace SqlStreamStore.Infrastructure using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; + using MorseCode.ITask; using SqlStreamStore.Logging; using SqlStreamStore.Streams; using SqlStreamStore.Subscriptions; @@ -13,7 +14,7 @@ namespace SqlStreamStore.Infrastructure /// /// Represents a base implementation of a readonly stream store. /// - public abstract class ReadonlyStreamStoreBase : IReadonlyStreamStore + public abstract class ReadonlyStreamStoreBase : IReadonlyStreamStore where TReadAllPage : IReadAllPage { private const int DefaultReloadInterval = 3000; protected readonly GetUtcNow GetUtcNow; @@ -42,7 +43,7 @@ protected ReadonlyStreamStoreBase(GetUtcNow getUtcNow, string logName) _disableMetadataCache = true; } - public async Task ReadAllForwards( + public async ITask ReadAllForwards( long fromPositionInclusive, int maxCount, bool prefetchJsonData, @@ -57,11 +58,124 @@ public async Task ReadAllForwards( Logger.DebugFormat("ReadAllForwards from position {fromPositionInclusive} with max count " + "{maxCount}.", fromPositionInclusive, maxCount); - Task ReadNext(long nextPosition, CancellationToken ct) => ReadAllForwards(nextPosition, maxCount, prefetchJsonData, ct); + //Task ReadNext(long nextPosition, CancellationToken ct) => ReadAllForwards(nextPosition, maxCount, prefetchJsonData, ct); - var page = await ReadAllForwardsInternal(fromPositionInclusive, maxCount, prefetchJsonData, ReadNext, cancellationToken) + var page = await ReadAllForwardsInternal(fromPositionInclusive, maxCount, prefetchJsonData, cancellationToken) .ConfigureAwait(false); + var pageHandled = await HandleGap(page, fromPositionInclusive, maxCount, prefetchJsonData, cancellationToken); + + return await FilterExpired(pageHandled, cancellationToken).ConfigureAwait(false); + + //// https://github.com/damianh/SqlStreamStore/issues/31 + //// Under heavy parallel load, gaps may appear in the position sequence due to sequence + //// number reservation of in-flight transactions. + //// Here we check if there are any gaps, and in the unlikely event there is, we delay a little bit + //// and re-issue the read. This is expected + //if(!page.IsEnd || page.Messages.Length <= 1) + //{ + // return await FilterExpired(page, ReadNext, cancellationToken).ConfigureAwait(false); + //} + + // only short circuit now for empty pages or 'old' pages, where + // 'old' is defined as pages whose last message is older than _positionWriteDelayThreshold ago + //if (page.Messages.Length == 0 || ((DateTime.UtcNow - page.Messages[page.Messages.Length - 1].CreatedUtc) > TimeSpan.FromMinutes(5))) + // return await FilterExpired(page, ReadNext, cancellationToken).ConfigureAwait(false); + + + //// TODO: FIXIT + //// Check for gap between last page and this. + //if (page.Messages[0].Position != fromPositionInclusive) + //{ + // Logger.DebugFormat("Gap detected at lower page boundary. Potentially could have lost {lostMessageCount} events if the gap is transient", page.Messages[0].Position - fromPositionInclusive); + // page = await HandleGap(page, fromPositionInclusive, maxCount, prefetchJsonData, ReadNext, cancellationToken); + // //if (!page.IsEnd || page.Messages.Length == 1) + // // Logger.DebugFormat("Gap detected at lower page boundary. Potentially could have lost {lostMessageCount} events if the gap is transient", page.Messages[0].Position - fromPositionInclusive); + // //page = await ReloadAfterDelay(fromPositionInclusive, maxCount, prefetchJsonData, ReadNext, cancellationToken); + //} + + //// check for gap in messages collection + //for (int i = 0; i < page.Messages.Length - 1; i++) + //{ + // var expectedNextPosition = page.Messages[i].Position + 1; + // if (expectedNextPosition != page.Messages[i + 1].Position) + // { + // Logger.InfoFormat("Gap detected in " + (page.IsEnd ? "last" : "(NOT the last)") + " page. Returning partial page {fromPosition}-{toPosition}", fromPositionInclusive, fromPositionInclusive + i + 1); + + // ReadAllPage requeryPage; + // var maxPosition = page.Messages[page.Messages.Length - 1].Position; + // do + // { + // requeryPage = await ReadAllForwardsInternal(fromPositionInclusive, maxCount, prefetchJsonData, ReadNext, cancellationToken, maxPosition); + // } while (page.TxSnapshot.CurrentTxIds.Intersect(requeryPage.TxSnapshot.CurrentTxIds).Any()); + + // return await FilterExpired(requeryPage, ReadNext, cancellationToken).ConfigureAwait(false); + + // // switched this to return the partial page, then re-issue load starting at gap + // // this speeds up the retry instead of taking a 3 second delay immediately + // //var messagesBeforeGap = new StreamMessage[i+1]; + // //page.Messages.Take(i+1).ToArray().CopyTo(messagesBeforeGap, 0); + // //return new ReadAllPage(page.FromPosition, maxPosition, page.IsEnd, page.Direction, ReadNext, messagesBeforeGap); + // } + //} + + //return await FilterExpired(page, ReadNext, cancellationToken).ConfigureAwait(false); + } + + //protected virtual async Task HandleGap(T page, long fromPositionInclusive, int maxCount, bool prefetchJsonData, ReadNextAllPage readNext, CancellationToken cancellationToken) where T : ReadAllPage + //{ + // if (page.Messages.Length == 0 || DateTime.UtcNow - page.Messages[page.Messages.Length - 1].CreatedUtc > TimeSpan.FromMinutes(5)) + // return page; + + + // // TODO: FIXIT + // // Check for gap between last page and this. + // if (page.Messages[0].Position != fromPositionInclusive) + // { + // Logger.DebugFormat("Gap detected at lower page boundary. Potentially could have lost {lostMessageCount} events if the gap is transient", page.Messages[0].Position - fromPositionInclusive); + // page = await HandleGap(page, fromPositionInclusive, maxCount, prefetchJsonData, readNext, cancellationToken); + // //if (!page.IsEnd || page.Messages.Length == 1) + // // Logger.DebugFormat("Gap detected at lower page boundary. Potentially could have lost {lostMessageCount} events if the gap is transient", page.Messages[0].Position - fromPositionInclusive); + // //page = await ReloadAfterDelay(fromPositionInclusive, maxCount, prefetchJsonData, ReadNext, cancellationToken); + // } + + // // check for gap in messages collection + // for (int i = 0; i < page.Messages.Length - 1; i++) + // { + // var expectedNextPosition = page.Messages[i].Position + 1; + // if (expectedNextPosition != page.Messages[i + 1].Position) + // { + // Logger.InfoFormat("Gap detected in " + (page.IsEnd ? "last" : "(NOT the last)") + " page. Returning partial page {fromPosition}-{toPosition}", fromPositionInclusive, fromPositionInclusive + i + 1); + + // ReadAllPage requeryPage; + // var maxPosition = page.Messages[page.Messages.Length - 1].Position; + // do + // { + // requeryPage = await ReadAllForwardsInternal(fromPositionInclusive, maxCount, prefetchJsonData, readNext, cancellationToken, maxPosition); + // } while (page.TxSnapshot.CurrentTxIds.Intersect(requeryPage.TxSnapshot.CurrentTxIds).Any()); + + // return requeryPage; + + // // switched this to return the partial page, then re-issue load starting at gap + // // this speeds up the retry instead of taking a 3 second delay immediately + // //var messagesBeforeGap = new StreamMessage[i+1]; + // //page.Messages.Take(i+1).ToArray().CopyTo(messagesBeforeGap, 0); + // //return new ReadAllPage(page.FromPosition, maxPosition, page.IsEnd, page.Direction, ReadNext, messagesBeforeGap); + // } + // } + + // //ReadAllPage requeryPage; + // //var maxPosition = pageWithGap.Messages[pageWithGap.Messages.Length - 1].Position; + // //do + // //{ + // // requeryPage = await ReadAllForwardsInternal(fromPositionInclusive, maxCount, prefetchJsonData, readNext, cancellationToken, maxPosition); + // //} while (pageWithGap.TxSnapshot.CurrentTxIds.Intersect(requeryPage.TxSnapshot.CurrentTxIds).Any()); + + // return page; + //} + + protected virtual async Task HandleGap(TReadAllPage page, long fromPositionInclusive, int maxCount, bool prefetchJsonData, CancellationToken cancellationToken) + { // https://github.com/damianh/SqlStreamStore/issues/31 // Under heavy parallel load, gaps may appear in the position sequence due to sequence // number reservation of in-flight transactions. @@ -69,13 +183,13 @@ public async Task ReadAllForwards( // and re-issue the read. This is expected if(!page.IsEnd || page.Messages.Length <= 1) { - return await FilterExpired(page, ReadNext, cancellationToken).ConfigureAwait(false); + return page; } // Check for gap between last page and this. if (page.Messages[0].Position != fromPositionInclusive) { - page = await ReloadAfterDelay(fromPositionInclusive, maxCount, prefetchJsonData, ReadNext, cancellationToken); + return await ReloadAfterDelay(fromPositionInclusive, maxCount, prefetchJsonData, cancellationToken); } // check for gap in messages collection @@ -83,15 +197,14 @@ public async Task ReadAllForwards( { if(page.Messages[i].Position + 1 != page.Messages[i + 1].Position) { - page = await ReloadAfterDelay(fromPositionInclusive, maxCount, prefetchJsonData, ReadNext, cancellationToken); - break; + return await ReloadAfterDelay(fromPositionInclusive, maxCount, prefetchJsonData, cancellationToken); } } - return await FilterExpired(page, ReadNext, cancellationToken).ConfigureAwait(false); + return page; } - public async Task ReadAllBackwards( + public async ITask ReadAllBackwards( long fromPositionInclusive, int maxCount, bool prefetchJsonData, @@ -108,9 +221,9 @@ public async Task ReadAllBackwards( fromPositionInclusive, maxCount); - ReadNextAllPage readNext = (nextPosition, ct) => ReadAllBackwards(nextPosition, maxCount, prefetchJsonData, ct); - var page = await ReadAllBackwardsInternal(fromPositionInclusive, maxCount, prefetchJsonData, readNext, cancellationToken); - return await FilterExpired(page, readNext, cancellationToken); + //ReadNextAllPage readNext = (nextPosition, ct) => ReadAllBackwards(nextPosition, maxCount, prefetchJsonData, ct); + var page = await ReadAllBackwardsInternal(fromPositionInclusive, maxCount, prefetchJsonData, cancellationToken); + return await FilterExpired(page, cancellationToken); } public async Task ReadStreamForwards( @@ -286,18 +399,17 @@ public void Dispose() public event Action OnDispose; - protected abstract Task ReadAllForwardsInternal( + protected abstract Task ReadAllForwardsInternal( long fromPositionExclusive, int maxCount, bool prefetch, - ReadNextAllPage readNext, - CancellationToken cancellationToken); + CancellationToken cancellationToken, + long fromMaxPositionInclusive = -1); - protected abstract Task ReadAllBackwardsInternal( + protected abstract Task ReadAllBackwardsInternal( long fromPositionExclusive, int maxCount, bool prefetch, - ReadNextAllPage readNext, CancellationToken cancellationToken); protected abstract Task ReadStreamForwardsInternal( @@ -348,11 +460,11 @@ protected abstract Task ListStreamsInternal( CancellationToken cancellationToken); protected virtual void Dispose(bool disposing) - {} + { } protected void GuardAgainstDisposed() { - if(_isDisposed) + if (_isDisposed) { throw new ObjectDisposedException(GetType().Name); } @@ -360,18 +472,17 @@ protected void GuardAgainstDisposed() protected abstract void PurgeExpiredMessage(StreamMessage streamMessage); - private async Task ReloadAfterDelay( + private async Task ReloadAfterDelay( long fromPositionInclusive, int maxCount, bool prefetch, - ReadNextAllPage readNext, CancellationToken cancellationToken) { - Logger.Info("ReadAllForwards: gap detected in position, reloading after {DefaultReloadInterval}ms", DefaultReloadInterval); - await Task.Delay(DefaultReloadInterval, cancellationToken); - var reloadedPage = await ReadAllForwardsInternal(fromPositionInclusive, maxCount, prefetch, readNext, cancellationToken) + Logger.Info("ReadAllForwards: gap detected in position, reloading after {DefaultReloadInterval}ms, position: {fromPositionInclusive}", DefaultReloadInterval, fromPositionInclusive); + //await Task.Delay(DefaultReloadInterval, cancellationToken); + var reloadedPage = await ReadAllForwardsInternal(fromPositionInclusive, maxCount, prefetch, cancellationToken) .ConfigureAwait(false); - return await FilterExpired(reloadedPage, readNext, cancellationToken).ConfigureAwait(false); + return await FilterExpired(reloadedPage, cancellationToken).ConfigureAwait(false); } private async Task FilterExpired( @@ -379,7 +490,7 @@ private async Task FilterExpired( ReadNextStreamPage readNext, CancellationToken cancellationToken) { - if(page.StreamId.StartsWith("$")) + if (page.StreamId.StartsWith("$")) { return page; } @@ -393,9 +504,9 @@ private async Task FilterExpired( } var currentUtc = GetUtcNow(); var valid = new List(); - foreach(var message in page.Messages) + foreach (var message in page.Messages) { - if(message.CreatedUtc.AddSeconds(maxAge.Value) > currentUtc) + if (message.CreatedUtc.AddSeconds(maxAge.Value) > currentUtc) { valid.Add(message); } @@ -417,12 +528,11 @@ private async Task FilterExpired( valid.ToArray()); } - private async Task FilterExpired( - ReadAllPage readAllPage, - ReadNextAllPage readNext, + private async Task FilterExpired(TReadAllPage readAllPage, CancellationToken cancellationToken) { - if(_disableMetadataCache) + return readAllPage; + if (_disableMetadataCache) { return readAllPage; } @@ -430,7 +540,7 @@ private async Task FilterExpired( var currentUtc = GetUtcNow(); foreach (var streamMessage in readAllPage.Messages) { - if(streamMessage.StreamId.StartsWith("$")) + if (streamMessage.StreamId.StartsWith("$")) { valid.Add(streamMessage); continue; @@ -452,13 +562,13 @@ private async Task FilterExpired( PurgeExpiredMessage(streamMessage); } } - return new ReadAllPage( - readAllPage.FromPosition, - readAllPage.NextPosition, - readAllPage.IsEnd, - readAllPage.Direction, - readNext, - valid.ToArray()); + //return new ReadAllPage( + // readAllPage.FromPosition, + // readAllPage.NextPosition, + // readAllPage.IsEnd, + // readAllPage.Direction, + // readNext, + // valid.ToArray()); } protected List FilterExpired(List<(StreamMessage StreamMessage, int? MaxAge)> messages) diff --git a/src/SqlStreamStore/Infrastructure/StreamStoreBase.cs b/src/SqlStreamStore/Infrastructure/StreamStoreBase.cs index b4b35c189..782505062 100644 --- a/src/SqlStreamStore/Infrastructure/StreamStoreBase.cs +++ b/src/SqlStreamStore/Infrastructure/StreamStoreBase.cs @@ -12,7 +12,7 @@ namespace SqlStreamStore.Infrastructure /// Represents a base implementation of a stream store. It's primary purpose is to handle /// common needs across all derived implementations such as guard clauses and logging. /// - public abstract class StreamStoreBase : ReadonlyStreamStoreBase, IStreamStore + public abstract class StreamStoreBase : ReadonlyStreamStoreBase, IStreamStore where TReadAllPage : IReadAllPage { private readonly TaskQueue _taskQueue = new TaskQueue(); diff --git a/src/SqlStreamStore/SqlStreamStore.csproj b/src/SqlStreamStore/SqlStreamStore.csproj index 56b7e7087..25e0c13a3 100644 --- a/src/SqlStreamStore/SqlStreamStore.csproj +++ b/src/SqlStreamStore/SqlStreamStore.csproj @@ -2,21 +2,20 @@ A library for writing and reading streams (also known as an 'event store') with a focus on SQL based implementations. This contains the core interfaces, abstractions and an in-memory implementation for tests. SQL Stream Store - netstandard2.0 + netstandard2.0 $(DefineConstants);LIBLOG_PUBLIC true SqlStreamStore SqlStreamStore - - - + all runtime; build; native; contentfiles; analyzers + \ No newline at end of file diff --git a/src/SqlStreamStore/StreamStoreExtensions.cs b/src/SqlStreamStore/StreamStoreExtensions.cs index d8840dcb2..fbbf93167 100644 --- a/src/SqlStreamStore/StreamStoreExtensions.cs +++ b/src/SqlStreamStore/StreamStoreExtensions.cs @@ -2,6 +2,7 @@ { using System.Threading; using System.Threading.Tasks; + using MorseCode.ITask; using SqlStreamStore.Streams; /// @@ -29,11 +30,11 @@ public static class StreamStoreExtensions /// An presenting the result of the read. If all messages read have expired /// then the message collection MAY be empty. /// - public static Task ReadAllForwards( - this IReadonlyStreamStore readonlyStreamStore, + public static ITask ReadAllForwards( + this IReadonlyStreamStore readonlyStreamStore, long fromPositionInclusive, int maxCount, - CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default) where TReadAllPage : IReadAllPage { return readonlyStreamStore.ReadAllForwards( fromPositionInclusive, @@ -61,11 +62,11 @@ public static Task ReadAllForwards( /// An presenting the result of the read. If all messages read have expired /// then the message collection MAY be empty. /// - public static Task ReadAllBackwards( - this IReadonlyStreamStore readonlyStreamStore, + public static ITask ReadAllBackwards( + this IReadonlyStreamStore readonlyStreamStore, long fromPositionInclusive, int maxCount, - CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default) where TReadAllPage : ReadAllPage, new() { return readonlyStreamStore.ReadAllBackwards( fromPositionInclusive, @@ -96,12 +97,12 @@ public static Task ReadAllBackwards( /// An represent the result of the operation. If all the messages read /// have expired then the message collection MAY be empty. /// - public static Task ReadStreamForwards( - this IReadonlyStreamStore readonlyStreamStore, + public static Task ReadStreamForwards( + this IReadonlyStreamStore readonlyStreamStore, string streamId, int fromVersionInclusive, int maxCount, - CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default) where TReadAllPage : IReadAllPage { return readonlyStreamStore.ReadStreamForwards( streamId, diff --git a/src/SqlStreamStore/Streams/ReadAllPage.cs b/src/SqlStreamStore/Streams/ReadAllPage.cs index 08181e718..f8557e284 100644 --- a/src/SqlStreamStore/Streams/ReadAllPage.cs +++ b/src/SqlStreamStore/Streams/ReadAllPage.cs @@ -1,39 +1,51 @@ namespace SqlStreamStore.Streams { - using System.Threading; - using System.Threading.Tasks; + public interface IReadAllPage + { + long FromPosition { get; } + long NextPosition { get; } + bool IsEnd { get; } + ReadDirection Direction { get; } + StreamMessage[] Messages { get; } + } + + public class ReadAllPage : ReadAllPage + { + public ReadAllPage(long fromPosition, long nextPosition, bool isEnd, ReadDirection direction, StreamMessage[] messages = null) : base(fromPosition, nextPosition, isEnd, direction, messages) + { } + } /// /// Represents the result of a read of all streams. /// - public sealed class ReadAllPage + public abstract class ReadAllPage : IReadAllPage where TReadAllPage : IReadAllPage { - private readonly ReadNextAllPage _readNext; + //private readonly ReadNextAllPage _readNext; /// /// A long representing the position where this page was read from. /// - public readonly long FromPosition; + public long FromPosition { get; } /// /// A long representing the position where the next page should be read from. /// - public readonly long NextPosition; + public long NextPosition { get; } /// /// True if page reach end of the all stream at time of reading. Otherwise false. /// - public readonly bool IsEnd; + public bool IsEnd { get; } /// /// The direction of the the read request. /// - public readonly ReadDirection Direction; + public ReadDirection Direction { get; } /// /// The collection of s returned as part of the read. /// - public readonly StreamMessage[] Messages; + public StreamMessage[] Messages { get; } /// /// Initializes a new instance of @@ -42,21 +54,18 @@ public sealed class ReadAllPage /// A long representing the position where the next page should be read from. /// True if page reach end of the all stream at time of reading. Otherwise false. /// The direction of the the read request. - /// An operation to read the next page of messages. /// The collection messages read. - public ReadAllPage( + protected ReadAllPage( long fromPosition, long nextPosition, bool isEnd, ReadDirection direction, - ReadNextAllPage readNext, StreamMessage[] messages = null) { FromPosition = fromPosition; NextPosition = nextPosition; IsEnd = isEnd; Direction = direction; - _readNext = readNext; Messages = messages ?? new StreamMessage[0]; } @@ -67,14 +76,14 @@ public override string ToString() $"IsEnd: {IsEnd}, Direction: {Direction}, SteamEventCount: {Messages.Length}"; } - /// - /// Reads the next page. - /// - /// A token to cancel the operations. - /// A task the represents the asyncronous operation. - public Task ReadNext(CancellationToken cancellationToken = default) - { - return _readNext(NextPosition, cancellationToken); - } + ///// + ///// Reads the next page. + ///// + ///// A token to cancel the operations. + ///// A task the represents the asyncronous operation. + //public Task ReadNext(CancellationToken cancellationToken = default) + //{ + // return _readNext(NextPosition, cancellationToken); + //} } } \ No newline at end of file diff --git a/src/SqlStreamStore/Streams/ReadNextAllPage.cs b/src/SqlStreamStore/Streams/ReadNextAllPage.cs index b80ba8522..9b4d676a3 100644 --- a/src/SqlStreamStore/Streams/ReadNextAllPage.cs +++ b/src/SqlStreamStore/Streams/ReadNextAllPage.cs @@ -3,11 +3,15 @@ using System.Threading; using System.Threading.Tasks; + //using System.Threading; + //using System.Threading.Tasks; + /// /// Represents an operation to read the next all page. /// /// The position to read from. /// A token to cancel the operation. /// A task that represents the result of the operation.z - public delegate Task ReadNextAllPage(long nextPosition, CancellationToken cancellationToken); + //public delegate Task ReadNextAllPage(long nextPosition, CancellationToken cancellationToken) where TReadAllPage : ReadAllPage; + public delegate Task ReadNextAllPage(long nextPosition, CancellationToken cancellationToken) where TReadAllPage : IReadAllPage; } \ No newline at end of file diff --git a/src/SqlStreamStore/Subscriptions/AllStreamSubscription.cs b/src/SqlStreamStore/Subscriptions/AllStreamSubscription.cs index 32ae0f214..c16c1de1e 100644 --- a/src/SqlStreamStore/Subscriptions/AllStreamSubscription.cs +++ b/src/SqlStreamStore/Subscriptions/AllStreamSubscription.cs @@ -13,13 +13,13 @@ /// Represents a subscription to all streams. /// - public sealed class AllStreamSubscription : IAllStreamSubscription + public sealed class AllStreamSubscription : IAllStreamSubscription where TReadAllPage : IReadAllPage { public const int DefaultPageSize = 10; private static readonly ILog s_logger = LogProvider.GetLogger("SqlStreamStore.Subscriptions.AllStreamSubscription"); private int _pageSize = DefaultPageSize; private long _nextPosition; - private readonly IReadonlyStreamStore _readonlyStreamStore; + private readonly IReadonlyStreamStore _readonlyStreamStore; private readonly AllStreamMessageReceived _streamMessageReceived; private readonly bool _prefetchJsonData; private readonly HasCaughtUp _hasCaughtUp; @@ -32,7 +32,7 @@ public sealed class AllStreamSubscription : IAllStreamSubscription public AllStreamSubscription( long? continueAfterPosition, - IReadonlyStreamStore readonlyStreamStore, + IReadonlyStreamStore readonlyStreamStore, IObservable streamStoreAppendedNotification, AllStreamMessageReceived streamMessageReceived, AllSubscriptionDropped subscriptionDropped, @@ -176,9 +176,9 @@ private async Task Initialize() _nextPosition = headPosition == 0 ? 0 : headPosition + 1; } - private async Task Pull() + private async Task Pull() { - ReadAllPage readAllPage; + TReadAllPage readAllPage; try { readAllPage = await _readonlyStreamStore @@ -204,7 +204,7 @@ private async Task Pull() return readAllPage; } - private async Task Push(ReadAllPage page) + private async Task Push(TReadAllPage page) { foreach (var message in page.Messages) { diff --git a/src/SqlStreamStore/Subscriptions/CreateStreamStoreNotifier.cs b/src/SqlStreamStore/Subscriptions/CreateStreamStoreNotifier.cs index 9cca6b22a..5be05e9c3 100644 --- a/src/SqlStreamStore/Subscriptions/CreateStreamStoreNotifier.cs +++ b/src/SqlStreamStore/Subscriptions/CreateStreamStoreNotifier.cs @@ -1,6 +1,6 @@ namespace SqlStreamStore.Subscriptions { - using SqlStreamStore; + using SqlStreamStore.Streams; /// /// Represents an operation to create a stream store notifier. diff --git a/tests/Directory.Build.props b/tests/Directory.Build.props index 1c0923a44..5f847107e 100644 --- a/tests/Directory.Build.props +++ b/tests/Directory.Build.props @@ -2,13 +2,13 @@ $(NoWarn);1701;1702;1705;1591;CS0618 - 7.3 + 9.0 true - - + + all diff --git a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.AppendStream.cs b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.AppendStream.cs index 49b41f47c..4eeb35f65 100644 --- a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.AppendStream.cs +++ b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.AppendStream.cs @@ -7,7 +7,7 @@ using SqlStreamStore.Streams; using Xunit; - public partial class AcceptanceTests + public partial class AcceptanceTests { [Fact, Trait("Category", "AppendStream")] public async Task diff --git a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.DeleteEvent.cs b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.DeleteEvent.cs index 49b10f92f..09e6c1d45 100644 --- a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.DeleteEvent.cs +++ b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.DeleteEvent.cs @@ -8,7 +8,7 @@ using Xunit; using static Streams.Deleted; - public partial class AcceptanceTests + public partial class AcceptanceTests { [Fact, Trait("Category", "DeleteEvent")] public async Task When_delete_message_then_message_should_be_removed_from_stream() diff --git a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.DeleteStream.cs b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.DeleteStream.cs index 4c5a1b549..bc13c7679 100644 --- a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.DeleteStream.cs +++ b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.DeleteStream.cs @@ -8,7 +8,7 @@ using Xunit; using static Streams.Deleted; - public partial class AcceptanceTests + public partial class AcceptanceTests { [Fact, Trait("Category", "DeleteStream")] public async Task When_delete_stream_with_no_expected_version_and_read_then_should_get_StreamNotFound() diff --git a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.DeletionTracking.cs b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.DeletionTracking.cs index 4e23ece16..20ad86899 100644 --- a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.DeletionTracking.cs +++ b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.DeletionTracking.cs @@ -6,7 +6,7 @@ namespace SqlStreamStore using Xunit; using static Streams.Deleted; - partial class AcceptanceTests + partial class AcceptanceTests { [Fact] public async Task When_deletion_tracking_is_disabled_deleted_message_should_not_be_tracked() @@ -25,7 +25,7 @@ public async Task When_deletion_tracking_is_disabled_deleted_message_should_not_ public async Task When_deletion_tracking_is_disabled_deleted_stream_should_not_be_tracked() { Fixture.DisableDeletionTracking = true; - + var messages = CreateNewStreamMessages(1); await Store.AppendToStream("stream", ExpectedVersion.NoStream, messages); await Store.DeleteStream("stream"); diff --git a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ListStreams.cs b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ListStreams.cs index c84f1fed6..2945c4599 100644 --- a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ListStreams.cs +++ b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ListStreams.cs @@ -7,7 +7,7 @@ namespace SqlStreamStore using SqlStreamStore.Streams; using Xunit; - public partial class AcceptanceTests + public partial class AcceptanceTests { [Theory] [InlineData("stream")] diff --git a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ReadAll.cs b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ReadAll.cs index ef068c035..79f6c0983 100644 --- a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ReadAll.cs +++ b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ReadAll.cs @@ -8,7 +8,7 @@ using SqlStreamStore.Streams; using Xunit; - public partial class AcceptanceTests + public partial class AcceptanceTests { [Fact, Trait("Category", "ReadAll")] public async Task Can_read_all_forwards() @@ -31,7 +31,7 @@ public async Task Can_read_all_forwards() int count = 0; while(!page.IsEnd && count <20) //should not take more than 20 iterations. { - page = await page.ReadNext(); + page = await Store.ReadAllForwards(page.NextPosition, 500); messages.AddRange(page.Messages); count++; } diff --git a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ReadHeadCheckpoint.cs b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ReadHeadCheckpoint.cs index 5481de59e..dd08440fc 100644 --- a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ReadHeadCheckpoint.cs +++ b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ReadHeadCheckpoint.cs @@ -5,7 +5,7 @@ using SqlStreamStore.Streams; using Xunit; - public partial class AcceptanceTests + public partial class AcceptanceTests { [Fact] public async Task Given_empty_store_when_get_head_position_Then_should_be_minus_one() diff --git a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ReadStream.cs b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ReadStream.cs index a79b5ae29..cc55114f1 100644 --- a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ReadStream.cs +++ b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ReadStream.cs @@ -10,7 +10,7 @@ using SqlStreamStore.Streams; using Xunit; - public partial class AcceptanceTests + public partial class AcceptanceTests { [Theory, Trait("Category", "ReadStream")] [MemberData(nameof(GetReadStreamForwardsTheories))] diff --git a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.StreamLimits.cs b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.StreamLimits.cs index 4f70de6e6..7e787f700 100644 --- a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.StreamLimits.cs +++ b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.StreamLimits.cs @@ -7,7 +7,7 @@ namespace SqlStreamStore using SqlStreamStore.Streams; using Xunit; - public partial class AcceptanceTests + public partial class AcceptanceTests { [Theory, Trait("Category", "StreamMetadata"), InlineData(ExpectedVersion.NoStream), InlineData(ExpectedVersion.Any)] @@ -63,7 +63,7 @@ await Store [Fact, Trait("Category", "StreamMetadata")] public async Task When_stream_has_expired_messages_and_read_forwards_then_should_not_get_expired_messages() { - var currentUtc = new DateTime(2016, 1, 1, 0, 0, 0); + var currentUtc = new DateTime(2016, 1, 1, 0, 0, 0, DateTimeKind.Utc); Fixture.GetUtcNow = () => currentUtc; string streamId = "stream-1"; await Store @@ -82,7 +82,7 @@ await Store [Fact, Trait("Category", "StreamMetadata")] public async Task When_stream_has_expired_messages_and_read_backward_then_should_not_get_expired_messages() { - var currentUtc = new DateTime(2016, 1, 1, 0, 0, 0); + var currentUtc = new DateTime(2016, 1, 1, 0, 0, 0, DateTimeKind.Utc); Fixture.GetUtcNow = () => currentUtc; const string streamId = "stream-1"; await Store @@ -101,7 +101,7 @@ await Store [Fact, Trait("Category", "StreamMetadata")] public async Task When_streams_have_expired_messages_and_read_all_forwards_then_should_not_get_expired_messages() { - var currentUtc = new DateTime(2016, 1, 1, 0, 0, 0); + var currentUtc = new DateTime(2016, 1, 1, 0, 0, 0, DateTimeKind.Utc); Fixture.GetUtcNow = () => currentUtc; // Arrange const string streamId1 = "stream-1"; @@ -134,7 +134,7 @@ await Store [Fact, Trait("Category", "StreamMetadata")] public async Task When_streams_have_expired_messages_and_read_all_backwards_then_should_not_get_expired_messages() { - var currentUtc = new DateTime(2016, 1, 1, 0, 0, 0); + var currentUtc = new DateTime(2016, 1, 1, 0, 0, 0, DateTimeKind.Utc); Fixture.GetUtcNow = () => currentUtc; // Arrange diff --git a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.StreamMetadata.cs b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.StreamMetadata.cs index 64999f5b0..fa1d68685 100644 --- a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.StreamMetadata.cs +++ b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.StreamMetadata.cs @@ -7,7 +7,7 @@ using SqlStreamStore.Streams; using Xunit; - public partial class AcceptanceTests + public partial class AcceptanceTests { private const string DefaultStreamMetadataJson = @"{ ""meta"": ""meta"" }"; //TODO: Port some of the tests from AppendStream with regard to expected version to verify behavior of Get/SetStreamMetadata. diff --git a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.Subscriptions.cs b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.Subscriptions.cs index 4f31d433d..34fdba683 100644 --- a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.Subscriptions.cs +++ b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.Subscriptions.cs @@ -12,7 +12,7 @@ using SqlStreamStore.TestUtils; using Xunit; - public partial class AcceptanceTests + public partial class AcceptanceTests { [Fact, Trait("Category", "Subscriptions")] public async Task Can_subscribe_to_a_stream_from_start() diff --git a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.cs b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.cs index ed550e769..cb1ab6ea5 100644 --- a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.cs +++ b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.cs @@ -4,13 +4,14 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; + using MorseCode.ITask; using Shouldly; using SqlStreamStore.Streams; using SqlStreamStore.TestUtils; using Xunit; using Xunit.Abstractions; - public abstract partial class AcceptanceTests : IAsyncLifetime + public abstract partial class AcceptanceTests : IAsyncLifetime where TReadAllPage : IReadAllPage { private const string DefaultJsonData = @"{ ""data"": ""data"" }"; private const string DefaultJsonMetadata = @"{ ""meta"": ""data"" }"; @@ -27,9 +28,9 @@ public async Task InitializeAsync() Fixture = await CreateFixture(); } - private IStreamStore Store => Fixture.Store; + private IStreamStore Store => Fixture.Store; - protected IStreamStoreFixture Fixture { get; private set; } + protected IStreamStoreFixture Fixture { get; private set; } protected ITestOutputHelper TestOutputHelper { get; } @@ -40,7 +41,7 @@ public Task DisposeAsync() return Task.CompletedTask; } - protected abstract Task CreateFixture(); + protected abstract Task> CreateFixture(); private static IDisposable CaptureLogs(ITestOutputHelper testOutputHelper) => LoggingHelper.Capture(testOutputHelper); @@ -50,7 +51,7 @@ public async Task When_dispose_and_read_then_should_throw() { Store.Dispose(); - Func act = () => Store.ReadAllForwards(Position.Start, 10); + Func act = () => Store.ReadAllForwards(Position.Start, 10).AsTask(); await act.ShouldThrowAsync(); } diff --git a/tests/SqlStreamStore.AcceptanceTests/IStreamStoreFixture.cs b/tests/SqlStreamStore.AcceptanceTests/IStreamStoreFixture.cs index af43de944..1b1a7231e 100644 --- a/tests/SqlStreamStore.AcceptanceTests/IStreamStoreFixture.cs +++ b/tests/SqlStreamStore.AcceptanceTests/IStreamStoreFixture.cs @@ -2,10 +2,11 @@ namespace SqlStreamStore { using System; using SqlStreamStore.Infrastructure; + using SqlStreamStore.Streams; - public interface IStreamStoreFixture: IDisposable + public interface IStreamStoreFixture : IDisposable where TReadPage : IReadAllPage { - IStreamStore Store { get; } + IStreamStore Store { get; } GetUtcNow GetUtcNow { get; set; } diff --git a/tests/SqlStreamStore.HAL.Tests/SqlStreamStore.HAL.Tests.csproj b/tests/SqlStreamStore.HAL.Tests/SqlStreamStore.HAL.Tests.csproj index 71272c22f..bc504238d 100644 --- a/tests/SqlStreamStore.HAL.Tests/SqlStreamStore.HAL.Tests.csproj +++ b/tests/SqlStreamStore.HAL.Tests/SqlStreamStore.HAL.Tests.csproj @@ -1,11 +1,11 @@  - netcoreapp2.1 + net6.0 SqlStreamStore.HAL.Tests SqlStreamStore.HAL.Tests - + diff --git a/tests/SqlStreamStore.Http.Tests/SqlStreamStore.Http.Tests.csproj b/tests/SqlStreamStore.Http.Tests/SqlStreamStore.Http.Tests.csproj index 6ebc9faf6..e7b5343c9 100644 --- a/tests/SqlStreamStore.Http.Tests/SqlStreamStore.Http.Tests.csproj +++ b/tests/SqlStreamStore.Http.Tests/SqlStreamStore.Http.Tests.csproj @@ -1,7 +1,7 @@  - netcoreapp3.1 + net6.0 SqlStreamStore.Http.Tests SqlStreamStore.Http.Tests SqlStreamStore @@ -19,6 +19,6 @@ - + diff --git a/tests/SqlStreamStore.MsSql.Tests/ExploratoryTests.cs b/tests/SqlStreamStore.MsSql.Tests/ExploratoryTests.cs index 04935b0b3..6c8857278 100644 --- a/tests/SqlStreamStore.MsSql.Tests/ExploratoryTests.cs +++ b/tests/SqlStreamStore.MsSql.Tests/ExploratoryTests.cs @@ -5,7 +5,7 @@ using SqlStreamStore.Streams; using Xunit; - public partial class AcceptanceTests + public partial class AcceptanceTests { [Fact] public async Task Time_to_take_to_read_1000_read_head_positions() diff --git a/tests/SqlStreamStore.MsSql.Tests/MsSqlStreamStoreAcceptanceTests.cs b/tests/SqlStreamStore.MsSql.Tests/MsSqlStreamStoreAcceptanceTests.cs index b87181281..fa415bc4a 100644 --- a/tests/SqlStreamStore.MsSql.Tests/MsSqlStreamStoreAcceptanceTests.cs +++ b/tests/SqlStreamStore.MsSql.Tests/MsSqlStreamStoreAcceptanceTests.cs @@ -1,10 +1,11 @@ namespace SqlStreamStore { using System.Threading.Tasks; + using SqlStreamStore.Streams; using Xunit; using Xunit.Abstractions; - public class MsSqlStreamStoreAcceptanceTests : AcceptanceTests, IClassFixture + public class MsSqlStreamStoreAcceptanceTests : AcceptanceTests, IClassFixture { private readonly MsSqlStreamStoreFixturePool _fixturePool; @@ -14,7 +15,7 @@ public MsSqlStreamStoreAcceptanceTests(MsSqlStreamStoreFixturePool fixturePool, _fixturePool = fixturePool; } - protected override async Task CreateFixture() + protected override async Task> CreateFixture() => await _fixturePool.Get(TestOutputHelper); /* diff --git a/tests/SqlStreamStore.MsSql.Tests/MsSqlStreamStoreFixture.cs b/tests/SqlStreamStore.MsSql.Tests/MsSqlStreamStoreFixture.cs index 8faa60f33..4754f35b0 100644 --- a/tests/SqlStreamStore.MsSql.Tests/MsSqlStreamStoreFixture.cs +++ b/tests/SqlStreamStore.MsSql.Tests/MsSqlStreamStoreFixture.cs @@ -3,9 +3,10 @@ namespace SqlStreamStore using System; using System.Threading.Tasks; using SqlStreamStore.Infrastructure; + using SqlStreamStore.Streams; using SqlStreamStore.TestUtils.MsSql; - public class MsSqlStreamStoreFixture : IStreamStoreFixture + public class MsSqlStreamStoreFixture : IStreamStoreFixture { private readonly Action _onDispose; private readonly MsSqlStreamStoreSettings _settings; @@ -33,7 +34,7 @@ public MsSqlStreamStoreFixture( public string DatabaseName { get; } - public IStreamStore Store => MsSqlStreamStore; + public IStreamStore Store => MsSqlStreamStore; public MsSqlStreamStore MsSqlStreamStore { get; private set; } diff --git a/tests/SqlStreamStore.MsSql.Tests/SqlStreamStore.MsSql.Tests.csproj b/tests/SqlStreamStore.MsSql.Tests/SqlStreamStore.MsSql.Tests.csproj index a7a500e40..236973360 100644 --- a/tests/SqlStreamStore.MsSql.Tests/SqlStreamStore.MsSql.Tests.csproj +++ b/tests/SqlStreamStore.MsSql.Tests/SqlStreamStore.MsSql.Tests.csproj @@ -1,6 +1,6 @@  - netcoreapp3.1 + net6.0 SqlStreamStore.MsSql.Tests SqlStreamStore.MsSql.Tests SqlStreamStore @@ -12,5 +12,4 @@ - \ No newline at end of file diff --git a/tests/SqlStreamStore.MsSql.Tests/StreamStoreAcceptanceTests.ReadLongStreamInPages.cs b/tests/SqlStreamStore.MsSql.Tests/StreamStoreAcceptanceTests.ReadLongStreamInPages.cs index b30f18ee7..31a16d4a9 100644 --- a/tests/SqlStreamStore.MsSql.Tests/StreamStoreAcceptanceTests.ReadLongStreamInPages.cs +++ b/tests/SqlStreamStore.MsSql.Tests/StreamStoreAcceptanceTests.ReadLongStreamInPages.cs @@ -8,7 +8,7 @@ namespace SqlStreamStore using SqlStreamStore.Streams; using Xunit; - public partial class AcceptanceTests + public partial class AcceptanceTests { [Fact] public async Task Given_large_message_stream_can_be_read_back_in_pages() diff --git a/tests/SqlStreamStore.MsSql.V3.Tests/ExploratoryTests.cs b/tests/SqlStreamStore.MsSql.V3.Tests/ExploratoryTests.cs index 5bd9faafe..5dc0eccf2 100644 --- a/tests/SqlStreamStore.MsSql.V3.Tests/ExploratoryTests.cs +++ b/tests/SqlStreamStore.MsSql.V3.Tests/ExploratoryTests.cs @@ -5,7 +5,7 @@ using SqlStreamStore.Streams; using Xunit; - public partial class AcceptanceTests + public partial class AcceptanceTests { [Fact] public async Task Time_to_take_to_read_1000_read_head_positions() diff --git a/tests/SqlStreamStore.MsSql.V3.Tests/MigrationTests.cs b/tests/SqlStreamStore.MsSql.V3.Tests/MigrationTests.cs index 1f5a04d0b..049f42c31 100644 --- a/tests/SqlStreamStore.MsSql.V3.Tests/MigrationTests.cs +++ b/tests/SqlStreamStore.MsSql.V3.Tests/MigrationTests.cs @@ -27,10 +27,10 @@ public async Task Can_migrate() var v2Store = v2Fixture.Store; await v2Store.AppendToStream("stream-1", ExpectedVersion.NoStream, - AcceptanceTests.CreateNewStreamMessages(1, 2, 3)); + AcceptanceTests.CreateNewStreamMessages(1, 2, 3)); await v2Store.AppendToStream("stream-2", ExpectedVersion.NoStream, - AcceptanceTests.CreateNewStreamMessages(1, 2, 3)); + AcceptanceTests.CreateNewStreamMessages(1, 2, 3)); await v2Store.SetStreamMetadata("stream-1", ExpectedVersion.Any, maxAge: 10, maxCount: 20); v2Store.Dispose(); diff --git a/tests/SqlStreamStore.MsSql.V3.Tests/MsSqlStreamStoreV3AcceptanceTests.cs b/tests/SqlStreamStore.MsSql.V3.Tests/MsSqlStreamStoreV3AcceptanceTests.cs index 6f9a19bd5..76b46ebab 100644 --- a/tests/SqlStreamStore.MsSql.V3.Tests/MsSqlStreamStoreV3AcceptanceTests.cs +++ b/tests/SqlStreamStore.MsSql.V3.Tests/MsSqlStreamStoreV3AcceptanceTests.cs @@ -9,7 +9,7 @@ using Xunit; using Xunit.Abstractions; - public class MsSqlStreamStoreV3AcceptanceTests : AcceptanceTests, IClassFixture + public class MsSqlStreamStoreV3AcceptanceTests : AcceptanceTests, IClassFixture { private readonly MsSqlStreamStoreV3FixturePool _fixturePool; @@ -19,7 +19,7 @@ public MsSqlStreamStoreV3AcceptanceTests(MsSqlStreamStoreV3FixturePool fixturePo _fixturePool = fixturePool; } - protected override async Task CreateFixture() + protected override async Task> CreateFixture() => await _fixturePool.Get(TestOutputHelper); [Fact] diff --git a/tests/SqlStreamStore.MsSql.V3.Tests/MsSqlStreamStoreV3Fixture.cs b/tests/SqlStreamStore.MsSql.V3.Tests/MsSqlStreamStoreV3Fixture.cs index bc6d8f7b3..adb993e2b 100644 --- a/tests/SqlStreamStore.MsSql.V3.Tests/MsSqlStreamStoreV3Fixture.cs +++ b/tests/SqlStreamStore.MsSql.V3.Tests/MsSqlStreamStoreV3Fixture.cs @@ -5,9 +5,10 @@ using System.Threading.Tasks; using Microsoft.Data.SqlClient; using SqlStreamStore.Infrastructure; + using SqlStreamStore.Streams; using SqlStreamStore.TestUtils.MsSql; - public class MsSqlStreamStoreV3Fixture : IStreamStoreFixture + public class MsSqlStreamStoreV3Fixture : IStreamStoreFixture { private readonly Action _onDispose; private readonly MsSqlStreamStoreV3Settings _settings; @@ -83,7 +84,7 @@ await command public string DatabaseName { get; } - public IStreamStore Store => MsSqlStreamStoreV3; + public IStreamStore Store => MsSqlStreamStoreV3; public MsSqlStreamStoreV3 MsSqlStreamStoreV3 { get; private set; } diff --git a/tests/SqlStreamStore.MsSql.V3.Tests/SqlStreamStore.MsSql.V3.Tests.csproj b/tests/SqlStreamStore.MsSql.V3.Tests/SqlStreamStore.MsSql.V3.Tests.csproj index 419e38c6e..234500363 100644 --- a/tests/SqlStreamStore.MsSql.V3.Tests/SqlStreamStore.MsSql.V3.Tests.csproj +++ b/tests/SqlStreamStore.MsSql.V3.Tests/SqlStreamStore.MsSql.V3.Tests.csproj @@ -1,6 +1,6 @@  - netcoreapp3.1 + net6.0 SqlStreamStore.MsSql.V3.Tests SqlStreamStore @@ -15,5 +15,4 @@ - \ No newline at end of file diff --git a/tests/SqlStreamStore.MsSql.V3.Tests/StreamStoreAcceptanceTests.ReadLongStreamInPages.cs b/tests/SqlStreamStore.MsSql.V3.Tests/StreamStoreAcceptanceTests.ReadLongStreamInPages.cs index b30f18ee7..31a16d4a9 100644 --- a/tests/SqlStreamStore.MsSql.V3.Tests/StreamStoreAcceptanceTests.ReadLongStreamInPages.cs +++ b/tests/SqlStreamStore.MsSql.V3.Tests/StreamStoreAcceptanceTests.ReadLongStreamInPages.cs @@ -8,7 +8,7 @@ namespace SqlStreamStore using SqlStreamStore.Streams; using Xunit; - public partial class AcceptanceTests + public partial class AcceptanceTests { [Fact] public async Task Given_large_message_stream_can_be_read_back_in_pages() diff --git a/tests/SqlStreamStore.MySql.Tests/SqlStreamStore.MySql.Tests.csproj b/tests/SqlStreamStore.MySql.Tests/SqlStreamStore.MySql.Tests.csproj index 00a35b777..657a35ece 100644 --- a/tests/SqlStreamStore.MySql.Tests/SqlStreamStore.MySql.Tests.csproj +++ b/tests/SqlStreamStore.MySql.Tests/SqlStreamStore.MySql.Tests.csproj @@ -1,6 +1,6 @@  - netcoreapp3.1 + net6.0 SqlStreamStore.MySql.Tests SqlStreamStore @@ -11,5 +11,4 @@ - diff --git a/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreAcceptanceTests.cs b/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreAcceptanceTests.cs index e09700c25..edea5fe04 100644 --- a/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreAcceptanceTests.cs +++ b/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreAcceptanceTests.cs @@ -4,7 +4,7 @@ using Xunit; using Xunit.Abstractions; - public class PostgresStreamStoreAcceptanceTests : AcceptanceTests, IClassFixture + public class PostgresStreamStoreAcceptanceTests : AcceptanceTests, IClassFixture { private readonly PostgresStreamStoreV3FixturePool _fixturePool; @@ -16,7 +16,7 @@ public PostgresStreamStoreAcceptanceTests( _fixturePool = fixturePool; } - protected override async Task CreateFixture() + protected override async Task> CreateFixture() => await _fixturePool.Get(TestOutputHelper); /* diff --git a/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreFixture.cs b/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreFixture.cs index 782de3a2e..2cab06b52 100644 --- a/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreFixture.cs +++ b/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreFixture.cs @@ -6,7 +6,7 @@ namespace SqlStreamStore using SqlStreamStore.Infrastructure; using SqlStreamStore.TestUtils.Postgres; - public class PostgresStreamStoreFixture : IStreamStoreFixture + public class PostgresStreamStoreFixture : IStreamStoreFixture { private readonly Action _onDispose; private bool _preparedPreviously; @@ -21,7 +21,7 @@ public PostgresStreamStoreFixture( _onDispose = onDispose; DatabaseName = databaseName; - var connectionString = dockerInstance.ConnectionString; + var connectionString = dockerInstance.GenerateConnectionString(); _settings = new PostgresStreamStoreSettings(connectionString) { @@ -41,7 +41,7 @@ public void Dispose() public string DatabaseName { get; } - public IStreamStore Store => PostgresStreamStore; + public IStreamStore Store => PostgresStreamStore; public PostgresStreamStore PostgresStreamStore { get; private set; } diff --git a/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreFixturePool.cs b/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreFixturePool.cs index ba5ed6f85..74a706329 100644 --- a/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreFixturePool.cs +++ b/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreFixturePool.cs @@ -23,7 +23,7 @@ public async Task Get( if (!fixturePool.TryDequeue(out var fixture)) { var databaseName = $"test_{Guid.NewGuid():n}"; - var dockerInstance = new PostgresContainer(databaseName); + var dockerInstance = new PostgresContainer(schema, databaseName); await dockerInstance.Start(); await dockerInstance.CreateDatabase(); @@ -31,7 +31,7 @@ public async Task Get( schema, dockerInstance, databaseName, - onDispose:() => fixturePool.Enqueue(fixture)); + () => fixturePool.Enqueue(fixture)); outputHelper.WriteLine($"Using new fixture with db {databaseName}"); } diff --git a/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreTests.cs b/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreTests.cs new file mode 100644 index 000000000..9f940a278 --- /dev/null +++ b/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreTests.cs @@ -0,0 +1,95 @@ +namespace SqlStreamStore +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using Shouldly; + using SqlStreamStore.Infrastructure; + using SqlStreamStore.Streams; + using SqlStreamStore.TestUtils; + using SqlStreamStore.TestUtils.Postgres; + using Xunit; + using Xunit.Abstractions; + + public class PostgresStreamStoreTests : IDisposable + { + private readonly PostgresContainer _container; + private const string Schema = "stresstest"; + private readonly ITestOutputHelper _testOutputHelper; + private readonly IDisposable _logCapture; + + public PostgresStreamStoreTests(ITestOutputHelper testOutputHelper) + { + _container = new PostgresContainer(Schema, $"test_{Guid.NewGuid():n}", 0.02f); + _testOutputHelper = testOutputHelper; + _logCapture = LoggingHelper.Capture(testOutputHelper); + } + + [Fact] + public async Task Can_Handle_High_Load_Gaps() + { + await _container.Start(); + await _container.CreateDatabase(); + + var readerSettings = GetSettings(_container.GenerateConnectionString("reader")); + var writerSettings = GetSettings(_container.GenerateConnectionString("writer")); + + using var readerStore = new PostgresStreamStore(readerSettings); + using var writerStore = new PostgresStreamStore(writerSettings); + + await writerStore.CreateSchemaIfNotExists(); + + var receiveMessages = new TaskCompletionSource(); + List receivedMessages = new List(); + + using var all = readerStore.SubscribeToAll( + Position.None, + (_, message, __) => + { + _testOutputHelper.WriteLine($"Received message {message.StreamId} " + + $"{message.StreamVersion} {message.Position}"); + if (message.Position >= 200) + receiveMessages.SetResult(message); + else + receivedMessages.Add(message); + + return Task.CompletedTask; + }); + + all.MaxCountPerRead = 500; + + await AppendMessages(writerStore, 1000); + await receiveMessages.Task.WithTimeout(1000000); + + receivedMessages.Count.ShouldBe(100); + } + + private static PostgresStreamStoreSettings GetSettings(string connectionString) + { + return new PostgresStreamStoreSettings(connectionString) + { + Schema = Schema, + GetUtcNow = SystemClock.GetUtcNow, + DisableDeletionTracking = false, + ScavengeAsynchronously = false + }; + } + + private static async Task AppendMessages(IStreamStore streamStore, int numberOfEvents) + { + await Task.WhenAll(Enumerable.Range(0, numberOfEvents).Select(_ => + { + var newMessage = new NewStreamMessage(Guid.NewGuid(), "MyEvent", "{}"); + return streamStore.AppendToStream(Guid.NewGuid().ToString(), ExpectedVersion.Any, newMessage); + })); + } + + public void Dispose() + { + _container.Dispose(); + _logCapture.Dispose(); + } + } +} \ No newline at end of file diff --git a/tests/SqlStreamStore.Postgres.Tests/SqlStreamStore.Postgres.Tests.csproj b/tests/SqlStreamStore.Postgres.Tests/SqlStreamStore.Postgres.Tests.csproj index 23e02cc30..52a732a77 100644 --- a/tests/SqlStreamStore.Postgres.Tests/SqlStreamStore.Postgres.Tests.csproj +++ b/tests/SqlStreamStore.Postgres.Tests/SqlStreamStore.Postgres.Tests.csproj @@ -1,7 +1,7 @@  - netcoreapp3.1 + net6.0 SqlStreamStore.Postgres.Tests SqlStreamStore diff --git a/tests/SqlStreamStore.Sqlite.Tests/SqlStreamStore.Sqlite.Tests.csproj b/tests/SqlStreamStore.Sqlite.Tests/SqlStreamStore.Sqlite.Tests.csproj index 6c5ebb697..185025726 100644 --- a/tests/SqlStreamStore.Sqlite.Tests/SqlStreamStore.Sqlite.Tests.csproj +++ b/tests/SqlStreamStore.Sqlite.Tests/SqlStreamStore.Sqlite.Tests.csproj @@ -1,7 +1,8 @@ - netcoreapp3.1 SqlStreamStore.Sqlite.Tests + net6.0 + SqlStreamStore.Sqlite.Tests SqlStreamStore @@ -11,5 +12,4 @@ - diff --git a/tests/SqlStreamStore.Tests/SqlStreamStore.Tests.csproj b/tests/SqlStreamStore.Tests/SqlStreamStore.Tests.csproj index d49b84447..f472007dc 100644 --- a/tests/SqlStreamStore.Tests/SqlStreamStore.Tests.csproj +++ b/tests/SqlStreamStore.Tests/SqlStreamStore.Tests.csproj @@ -1,6 +1,6 @@  - netcoreapp3.1 + net6.0 SqlStreamStore.Tests SqlStreamStore diff --git a/tools/hal-docs/gen-hal-docs.ps1 b/tools/hal-docs/gen-hal-docs.ps1 index 2c9c3f455..eb36aae4d 100644 --- a/tools/hal-docs/gen-hal-docs.ps1 +++ b/tools/hal-docs/gen-hal-docs.ps1 @@ -3,5 +3,5 @@ pushd cd ../../ -dotnet run -p build/build.csproj -- build-hal-docs +dotnet run --project build/build.csproj -- build-hal-docs popd \ No newline at end of file