From 194045fc6b324f2eac870dca288a8853e87fa8bd Mon Sep 17 00:00:00 2001 From: Arne Schoonvliet Date: Wed, 13 Oct 2021 15:05:09 +0200 Subject: [PATCH 01/13] Working on fixing sqlstreamstore pgsql --- .../PgSqlScripts/ReadAll.sql | 43 ++++--- .../PgSqlScripts/Schema.cs | 2 +- .../PostgresStreamStore.ReadAll.cs | 106 +++++++++++------- .../SqlStreamStore.TestUtils.csproj | 2 +- .../Infrastructure/ReadonlyStreamStoreBase.cs | 63 ++++++++--- src/SqlStreamStore/Streams/ReadAllPage.cs | 28 ++++- 6 files changed, 172 insertions(+), 72 deletions(-) diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/ReadAll.sql b/src/SqlStreamStore.Postgres/PgSqlScripts/ReadAll.sql index ea3ad5ffa..c036e245a 100644 --- a/src/SqlStreamStore.Postgres/PgSqlScripts/ReadAll.sql +++ b/src/SqlStreamStore.Postgres/PgSqlScripts/ReadAll.sql @@ -1,24 +1,34 @@ -CREATE OR REPLACE FUNCTION __schema__.read_all( +CREATE OR REPLACE FUNCTION __schema__.read_all2( _count INT, _position BIGINT, + _max_position BIGINT, _forwards BOOLEAN, _prefetch BOOLEAN ) - RETURNS TABLE( - stream_id VARCHAR(1000), - message_id UUID, - stream_version INT, - "position" BIGINT, - create_utc TIMESTAMP, - "type" VARCHAR(128), - json_metadata JSONB, - json_data JSONB, - max_age INT - ) + -- RETURNS TABLE( + -- stream_id VARCHAR(1000), + -- message_id UUID, + -- stream_version INT, + -- "position" BIGINT, + -- create_utc TIMESTAMP, + -- "type" VARCHAR(128), + -- json_metadata JSONB, + -- json_data JSONB, + -- max_age INT + -- ) + RETURNS SETOF REFCURSOR + AS $F$ + +DECLARE + -- _stream_id_internal INT; + -- _stream_info REFCURSOR := 'stream_info'; + _messages REFCURSOR := 'messages'; + _txinfo REFCURSOR := 'tx_info'; + BEGIN - RETURN QUERY + OPEN _messages FOR WITH messages AS ( SELECT __schema__.streams.id_original, __schema__.messages.message_id, @@ -34,6 +44,7 @@ BEGIN FROM __schema__.messages INNER JOIN __schema__.streams ON __schema__.messages.stream_id_internal = __schema__.streams.id_internal WHERE (CASE + WHEN _forwards && _max_position > 0 THEN __schema__.messages.position >= _position AND __schema__.messages.position <= _max_position WHEN _forwards THEN __schema__.messages.position >= _position ELSE __schema__.messages.position <= _position END) ORDER BY @@ -42,6 +53,12 @@ BEGIN LIMIT _count ) SELECT * FROM messages LIMIT _count; + + RETURN NEXT _messages; + + OPEN _txinfo FOR + SELECT txid_current_snapshot()::TEXT; + RETURN NEXT _txinfo; END; $F$ LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/Schema.cs b/src/SqlStreamStore.Postgres/PgSqlScripts/Schema.cs index deb1e09ab..fb2a5e916 100644 --- a/src/SqlStreamStore.Postgres/PgSqlScripts/Schema.cs +++ b/src/SqlStreamStore.Postgres/PgSqlScripts/Schema.cs @@ -20,7 +20,7 @@ internal class Schema public string ListStreamsStartingWith => $"{_schema}.list_streams_starting_with"; public string ListStreamsEndingWith => $"{_schema}.list_streams_ending_with"; public string Read => $"{_schema}.read"; - public string ReadAll => $"{_schema}.read_all"; + public string ReadAll => $"{_schema}.read_all2"; public string ReadAllHeadPosition => $"{_schema}.read_head_position"; public string ReadStreamHeadPosition => $"{_schema}.read_stream_head_position"; public string ReadStreamHeadVersion => $"{_schema}.read_stream_head_version"; diff --git a/src/SqlStreamStore.Postgres/PostgresStreamStore.ReadAll.cs b/src/SqlStreamStore.Postgres/PostgresStreamStore.ReadAll.cs index 1eeedf0b5..0e50fe6d6 100644 --- a/src/SqlStreamStore.Postgres/PostgresStreamStore.ReadAll.cs +++ b/src/SqlStreamStore.Postgres/PostgresStreamStore.ReadAll.cs @@ -4,8 +4,12 @@ using System.Collections.Generic; using System.Data; using System.Data.Common; + using System.Text; using System.Threading; using System.Threading.Tasks; + + using Npgsql; + using SqlStreamStore.PgSqlScripts; using SqlStreamStore.Streams; @@ -20,66 +24,90 @@ protected override async Task ReadAllForwardsInternal( { maxCount = maxCount == int.MaxValue ? maxCount - 1 : maxCount; + var refcursorSql = new StringBuilder(); + using(var connection = await OpenConnection(cancellationToken)) using(var transaction = connection.BeginTransaction()) - using(var command = BuildFunctionCommand( + { + using(var command = BuildFunctionCommand( _schema.ReadAll, transaction, Parameters.Count(maxCount + 1), Parameters.Position(fromPositionExclusive), Parameters.ReadDirection(ReadDirection.Forward), Parameters.Prefetch(prefetch))) - using(var reader = await command - .ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken) - .ConfigureAwait(false)) - { - if(!reader.HasRows) + using(var reader = await command + .ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken) + .ConfigureAwait(false)) { - return new ReadAllPage( - fromPositionExclusive, - fromPositionExclusive, - true, - ReadDirection.Forward, - readNext, - Array.Empty()); + while(await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + refcursorSql.AppendLine(Schema.FetchAll(reader.GetString(0))); + } } - var messages = new List<(StreamMessage message, int? maxAge)>(); - - while(await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + using(var command2 = new NpgsqlCommand(refcursorSql.ToString(), transaction.Connection, transaction)) + using(var reader2 = await command2 + .ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken) + .ConfigureAwait(false)) { - if(messages.Count == maxCount) + + if(!reader2.HasRows) { - messages.Add(default); + return new ReadAllPage( + fromPositionExclusive, + fromPositionExclusive, + true, + ReadDirection.Forward, + readNext, + Array.Empty()); } - else + + var messages = new List<(StreamMessage message, int? maxAge)>(); + + while(await reader2.ReadAsync(cancellationToken).ConfigureAwait(false)) { - var streamIdInfo = new StreamIdInfo(reader.GetString(0)); - var (message, maxAge, _) = - await ReadAllStreamMessage(reader, streamIdInfo.PostgresqlStreamId, prefetch); - messages.Add((message, maxAge)); + if(messages.Count == maxCount) + { + messages.Add(default); + } + else + { + var streamIdInfo = new StreamIdInfo(reader2.GetString(0)); + var (message, maxAge, _) = + await ReadAllStreamMessage(reader2, streamIdInfo.PostgresqlStreamId, prefetch); + messages.Add((message, maxAge)); + } } - } - bool isEnd = true; + string txSnapshot = string.Empty; + await reader2.NextResultAsync(cancellationToken).ConfigureAwait(false); + while (await reader2.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + txSnapshot = await reader2.GetFieldValueAsync(0); + } - if(messages.Count == maxCount + 1) // An extra row was read, we're not at the end - { - isEnd = false; - messages.RemoveAt(maxCount); - } + bool isEnd = true; - var filteredMessages = FilterExpired(messages); + if(messages.Count == maxCount + 1) // An extra row was read, we're not at the end + { + isEnd = false; + messages.RemoveAt(maxCount); + } - var nextPosition = filteredMessages[filteredMessages.Count - 1].Position + 1; + var filteredMessages = FilterExpired(messages); - return new ReadAllPage( - fromPositionExclusive, - nextPosition, - isEnd, - ReadDirection.Forward, - readNext, - filteredMessages.ToArray()); + var nextPosition = filteredMessages[filteredMessages.Count - 1].Position + 1; + + return new ReadAllPage( + fromPositionExclusive, + nextPosition, + isEnd, + ReadDirection.Forward, + readNext, + filteredMessages.ToArray(), + txSnapshot); + } } } diff --git a/src/SqlStreamStore.TestUtils/SqlStreamStore.TestUtils.csproj b/src/SqlStreamStore.TestUtils/SqlStreamStore.TestUtils.csproj index 4c7a7731b..67653edce 100644 --- a/src/SqlStreamStore.TestUtils/SqlStreamStore.TestUtils.csproj +++ b/src/SqlStreamStore.TestUtils/SqlStreamStore.TestUtils.csproj @@ -1,6 +1,6 @@  - netcoreapp2.1 + netcoreapp3.1 SqlStreamStore.TestUtils false diff --git a/src/SqlStreamStore/Infrastructure/ReadonlyStreamStoreBase.cs b/src/SqlStreamStore/Infrastructure/ReadonlyStreamStoreBase.cs index 1b20be17e..a692ccdde 100644 --- a/src/SqlStreamStore/Infrastructure/ReadonlyStreamStoreBase.cs +++ b/src/SqlStreamStore/Infrastructure/ReadonlyStreamStoreBase.cs @@ -1,3 +1,5 @@ +using System.Linq; + namespace SqlStreamStore.Infrastructure { using System; @@ -62,29 +64,55 @@ public async Task ReadAllForwards( var page = await ReadAllForwardsInternal(fromPositionInclusive, maxCount, prefetchJsonData, ReadNext, cancellationToken) .ConfigureAwait(false); - // https://github.com/damianh/SqlStreamStore/issues/31 - // Under heavy parallel load, gaps may appear in the position sequence due to sequence - // number reservation of in-flight transactions. - // Here we check if there are any gaps, and in the unlikely event there is, we delay a little bit - // and re-issue the read. This is expected - if(!page.IsEnd || page.Messages.Length <= 1) - { + + + //// https://github.com/damianh/SqlStreamStore/issues/31 + //// Under heavy parallel load, gaps may appear in the position sequence due to sequence + //// number reservation of in-flight transactions. + //// Here we check if there are any gaps, and in the unlikely event there is, we delay a little bit + //// and re-issue the read. This is expected + //if(!page.IsEnd || page.Messages.Length <= 1) + //{ + // return await FilterExpired(page, ReadNext, cancellationToken).ConfigureAwait(false); + //} + + // only short circuit now for empty pages or 'old' pages, where + // 'old' is defined as pages whose last message is older than _positionWriteDelayThreshold ago + if(page.Messages.Length == 0 || ((DateTime.UtcNow - page.Messages[page.Messages.Length-1].CreatedUtc) > TimeSpan.FromMinutes(5))) return await FilterExpired(page, ReadNext, cancellationToken).ConfigureAwait(false); - } - // Check for gap between last page and this. - if (page.Messages[0].Position != fromPositionInclusive) - { - page = await ReloadAfterDelay(fromPositionInclusive, maxCount, prefetchJsonData, ReadNext, cancellationToken); - } + + + //// Check for gap between last page and this. + //if(page.Messages[0].Position != fromPositionInclusive) + //{ + // if(!page.IsEnd || page.Messages.Length == 1) + // Logger.DebugFormat("Gap detected at lower page boundary. Potentially could have lost {lostMessageCount} events if the gap is transient", page.Messages[0].Position - fromPositionInclusive); + // page = await ReloadAfterDelay(fromPositionInclusive, maxCount, prefetchJsonData, ReadNext, cancellationToken); + //} // check for gap in messages collection for(int i = 0; i < page.Messages.Length - 1; i++) { - if(page.Messages[i].Position + 1 != page.Messages[i + 1].Position) + var expectedNextPosition = page.Messages[i].Position + 1; + if(expectedNextPosition != page.Messages[i + 1].Position) { - page = await ReloadAfterDelay(fromPositionInclusive, maxCount, prefetchJsonData, ReadNext, cancellationToken); - break; + Logger.InfoFormat("Gap detected in "+(page.IsEnd?"last":"(NOT the last)")+" page. Returning partial page {fromPosition}-{toPosition}", fromPositionInclusive, fromPositionInclusive+i+1); + + ReadAllPage requeryPage; + var maxPosition = page.Messages[page.Messages.Length - 1].Position; + do + { + requeryPage = await ReadAllForwardsInternal(fromPositionInclusive, maxCount, prefetchJsonData, ReadNext, cancellationToken, maxPosition); + } while(page.TxSnapshot.CurrentTxIds.Intersect(requeryPage.TxSnapshot.CurrentTxIds).Any()); + + return await FilterExpired(requeryPage, ReadNext, cancellationToken).ConfigureAwait(false); + + // switched this to return the partial page, then re-issue load starting at gap + // this speeds up the retry instead of taking a 3 second delay immediately + //var messagesBeforeGap = new StreamMessage[i+1]; + //page.Messages.Take(i+1).ToArray().CopyTo(messagesBeforeGap, 0); + //return new ReadAllPage(page.FromPosition, maxPosition, page.IsEnd, page.Direction, ReadNext, messagesBeforeGap); } } @@ -291,7 +319,8 @@ protected abstract Task ReadAllForwardsInternal( int maxCount, bool prefetch, ReadNextAllPage readNext, - CancellationToken cancellationToken); + CancellationToken cancellationToken, + long fromMaxPositionInclusive = -1); protected abstract Task ReadAllBackwardsInternal( long fromPositionExclusive, diff --git a/src/SqlStreamStore/Streams/ReadAllPage.cs b/src/SqlStreamStore/Streams/ReadAllPage.cs index 08181e718..4e20997c4 100644 --- a/src/SqlStreamStore/Streams/ReadAllPage.cs +++ b/src/SqlStreamStore/Streams/ReadAllPage.cs @@ -1,5 +1,8 @@ namespace SqlStreamStore.Streams { + using System; + using System.Collections.Generic; + using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -35,6 +38,8 @@ public sealed class ReadAllPage /// public readonly StreamMessage[] Messages; + public readonly TxSnapshot TxSnapshot; + /// /// Initializes a new instance of /// @@ -50,7 +55,8 @@ public ReadAllPage( bool isEnd, ReadDirection direction, ReadNextAllPage readNext, - StreamMessage[] messages = null) + StreamMessage[] messages = null, + string txSnapshot = null) { FromPosition = fromPosition; NextPosition = nextPosition; @@ -58,6 +64,7 @@ public ReadAllPage( Direction = direction; _readNext = readNext; Messages = messages ?? new StreamMessage[0]; + TxSnapshot = new TxSnapshot(txSnapshot); } /// @@ -77,4 +84,23 @@ public Task ReadNext(CancellationToken cancellationToken = default) return _readNext(NextPosition, cancellationToken); } } + + public class TxSnapshot + { + private readonly long MinTx; + private readonly long MaxTx; + + public readonly List CurrentTxIds = new List(); + public TxSnapshot(string txSnapshot) + { + var splitted = txSnapshot.Split(':'); + MinTx = Convert.ToInt64(splitted[0]); + MaxTx = Convert.ToInt64(splitted[1]); + + if(splitted.Length > 2 && !string.IsNullOrWhiteSpace(splitted[2])) + { + CurrentTxIds = splitted[2].Split(',').Select(x => Convert.ToInt64(x)).ToList(); + } + } + } } \ No newline at end of file From ab549334a0ebb4d93c3919bf8c8832107b310676 Mon Sep 17 00:00:00 2001 From: Arne Schoonvliet Date: Wed, 13 Oct 2021 15:53:59 +0200 Subject: [PATCH 02/13] Hacked solution for high cocurrent load --- src/LoadTests/LoadTest.cs | 39 - src/LoadTests/LoadTests.csproj | 4 +- src/LoadTests/MsSqlStreamStoreDb.cs | 86 -- src/LoadTests/MsSqlStreamStoreDbV3.cs | 96 --- src/LoadTests/MySqlStreamStoreDb.cs | 49 -- src/LoadTests/SqlLocalDb.cs | 62 -- src/LoadTests/TestGaps.cs | 3 +- .../PgSqlScripts/AppendToStream.sql | 5 +- .../PgSqlScripts/Read.sql | 1 + .../PgSqlScripts/ReadAll.sql | 7 +- .../PgSqlScripts/Tables.sql | 3 +- .../PostgresStreamStore.ReadAll.cs | 6 +- .../Postgres/PostgresContainer.cs | 3 +- .../InMemory/InMemoryAllStream.cs | 7 - src/SqlStreamStore/InMemory/InMemoryStream.cs | 225 ----- .../InMemory/InMemoryStreamMessage.cs | 36 - .../InMemory/InMemoryStreamStore.cs | 781 ------------------ .../Infrastructure/ReadonlyStreamStoreBase.cs | 4 +- src/SqlStreamStore/Streams/ReadAllPage.cs | 2 +- .../SqlStreamStore.HAL.Tests.csproj | 2 +- 20 files changed, 22 insertions(+), 1399 deletions(-) delete mode 100644 src/LoadTests/MsSqlStreamStoreDb.cs delete mode 100644 src/LoadTests/MsSqlStreamStoreDbV3.cs delete mode 100644 src/LoadTests/MySqlStreamStoreDb.cs delete mode 100644 src/LoadTests/SqlLocalDb.cs delete mode 100644 src/SqlStreamStore/InMemory/InMemoryAllStream.cs delete mode 100644 src/SqlStreamStore/InMemory/InMemoryStream.cs delete mode 100644 src/SqlStreamStore/InMemory/InMemoryStreamMessage.cs delete mode 100644 src/SqlStreamStore/InMemory/InMemoryStreamStore.cs diff --git a/src/LoadTests/LoadTest.cs b/src/LoadTests/LoadTest.cs index 3b47abd43..80c9954c6 100644 --- a/src/LoadTests/LoadTest.cs +++ b/src/LoadTests/LoadTest.cs @@ -17,38 +17,6 @@ public abstract class LoadTest Output.WriteLine(ConsoleColor.Yellow, "Store type:"); await new Menu() - .AddSync("InMem", () => streamStore = new InMemoryStreamStore()) - .Add("MS SQL V2 (Docker)", - async _ => - { - var fixture = new MsSqlStreamStoreDb("dbo"); - Console.WriteLine(fixture.ConnectionString); - streamStore = await fixture.GetStreamStore(); - disposable = fixture; - }) - .Add("MS SQL V3 (Docker)", - async _ => - { - var fixture = new MsSqlStreamStoreDbV3("dbo"); - Console.WriteLine(fixture.ConnectionString); - streamStore = await fixture.GetStreamStore(); - disposable = fixture; - }) - .AddSync("MS SQL V3 (LocalDB)", - () => - { - var sqlLocalDb = new SqlLocalDb(); - Console.WriteLine(sqlLocalDb.ConnectionString); - streamStore = sqlLocalDb.StreamStore; - disposable = sqlLocalDb; - }) - .Add("MYSQL V3 (Docker)", - async ct => - { - var mysqlDb = new MySqlStreamStoreDb(); - streamStore = await mysqlDb.GetMySqlStreamStore(); - disposable = streamStore; - }) .Add("Postgres (Docker)", async ct => { @@ -67,13 +35,6 @@ public abstract class LoadTest streamStore = await postgresStreamStoreDb.GetPostgresStreamStore(true); disposable = postgresStreamStoreDb; }) - .Add("MySql (Docker)", - async ct => - { - var db = new MySqlStreamStoreDb(); - streamStore = await db.GetMySqlStreamStore(); - disposable = db; - }) /*.Add("MySql (Server)", async ct => { diff --git a/src/LoadTests/LoadTests.csproj b/src/LoadTests/LoadTests.csproj index b6a85c474..c4095b798 100644 --- a/src/LoadTests/LoadTests.csproj +++ b/src/LoadTests/LoadTests.csproj @@ -1,4 +1,4 @@ - + netcoreapp3.1 win10-x64;osx.10.11-x64;ubuntu.16.10-x64;ubuntu.14.04-x64 @@ -15,8 +15,6 @@ - - diff --git a/src/LoadTests/MsSqlStreamStoreDb.cs b/src/LoadTests/MsSqlStreamStoreDb.cs deleted file mode 100644 index 50f54669a..000000000 --- a/src/LoadTests/MsSqlStreamStoreDb.cs +++ /dev/null @@ -1,86 +0,0 @@ -namespace LoadTests -{ - using System; - using System.Threading.Tasks; - using Microsoft.Data.SqlClient; - using SqlStreamStore; - using SqlStreamStore.TestUtils.MsSql; - - public class MsSqlStreamStoreDb : IDisposable - { - public readonly string ConnectionString; - private readonly string _schema; - private readonly bool _deleteDatabaseOnDispose; - private readonly string _databaseName; - private readonly SqlServerContainer _containerInstance; - - public MsSqlStreamStoreDb(string schema, bool deleteDatabaseOnDispose = true) - { - _schema = schema; - _deleteDatabaseOnDispose = deleteDatabaseOnDispose; - _databaseName = $"sss-v2-{Guid.NewGuid():n}"; - _containerInstance = new SqlServerContainer(_databaseName); - - ConnectionString = CreateConnectionString(); - } - - public async Task GetStreamStore() - { - await CreateDatabase(); - - return await GetStreamStore(_schema); - } - - public async Task GetStreamStore(string schema) - { - var settings = new MsSqlStreamStoreSettings(ConnectionString) - { - Schema = schema, - }; - var store = new MsSqlStreamStore(settings); - await store.CreateSchema(); - - return store; - } - - public void Dispose() - { - if (!_deleteDatabaseOnDispose) - { - return; - } - using (var sqlConnection = new SqlConnection(ConnectionString)) - { - // Fixes: "Cannot drop database because it is currently in use" - SqlConnection.ClearPool(sqlConnection); - } - using (var connection = _containerInstance.CreateConnection()) - { - connection.Open(); - using (var command = new SqlCommand($"ALTER DATABASE [{_databaseName}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE", connection)) - { - command.ExecuteNonQuery(); - } - using (var command = new SqlCommand($"DROP DATABASE [{_databaseName}]", connection)) - { - command.ExecuteNonQuery(); - } - } - } - - private async Task CreateDatabase() - { - await _containerInstance.Start(); - await _containerInstance.CreateDatabase(); - } - - private string CreateConnectionString() - { - var connectionStringBuilder = _containerInstance.CreateConnectionStringBuilder(); - connectionStringBuilder.MultipleActiveResultSets = true; - connectionStringBuilder.InitialCatalog = _databaseName; - - return connectionStringBuilder.ToString(); - } - } -} \ No newline at end of file diff --git a/src/LoadTests/MsSqlStreamStoreDbV3.cs b/src/LoadTests/MsSqlStreamStoreDbV3.cs deleted file mode 100644 index 0c951e251..000000000 --- a/src/LoadTests/MsSqlStreamStoreDbV3.cs +++ /dev/null @@ -1,96 +0,0 @@ -namespace LoadTests -{ - using System; - using System.Threading.Tasks; - using Microsoft.Data.SqlClient; - using SqlStreamStore; - using SqlStreamStore.TestUtils.MsSql; - - public class MsSqlStreamStoreDbV3 : IDisposable - { - public readonly string ConnectionString; - private readonly string _schema; - private readonly bool _disableDeletionTracking; - private readonly string _databaseNameOverride; - private readonly bool _deleteDatabaseOnDispose; - private readonly string _databaseName; - private readonly SqlServerContainer _containerInstance; - - public MsSqlStreamStoreDbV3( - string schema, - bool disableDeletionTracking = false, - string databaseNameOverride = null, - bool deleteDatabaseOnDispose = true) - { - _schema = schema; - _disableDeletionTracking = disableDeletionTracking; - _databaseNameOverride = databaseNameOverride; - _deleteDatabaseOnDispose = deleteDatabaseOnDispose; - _databaseName = databaseNameOverride ?? $"sss-v3-{Guid.NewGuid():n}"; - _containerInstance = new SqlServerContainer(_databaseName); - - ConnectionString = CreateConnectionString(); - } - - public async Task GetStreamStore() - { - await CreateDatabase(); - - return await GetStreamStore(_schema); - } - - private async Task GetStreamStore(string schema) - { - var settings = new MsSqlStreamStoreV3Settings(ConnectionString) - { - Schema = schema, - DisableDeletionTracking = _disableDeletionTracking - }; - var store = new MsSqlStreamStoreV3(settings); - await store.CreateSchemaIfNotExists(); - - return store; - } - - public void Dispose() - { - if (!_deleteDatabaseOnDispose) - { - return; - } - - SqlConnection.ClearAllPools(); - - using (var connection = _containerInstance.CreateConnection()) - { - connection.Open(); - using (var command = new SqlCommand($"ALTER DATABASE [{_databaseName}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE", connection)) - { - command.ExecuteNonQuery(); - } - using (var command = new SqlCommand($"DROP DATABASE [{_databaseName}]", connection)) - { - command.ExecuteNonQuery(); - } - } - } - - private async Task CreateDatabase() - { - if (_databaseNameOverride == null) - { - await _containerInstance.Start(); - await _containerInstance.CreateDatabase(); - } - } - - private string CreateConnectionString() - { - var connectionStringBuilder = _containerInstance.CreateConnectionStringBuilder(); - connectionStringBuilder.MultipleActiveResultSets = true; - connectionStringBuilder.InitialCatalog = _databaseName; - - return connectionStringBuilder.ToString(); - } - } -} \ No newline at end of file diff --git a/src/LoadTests/MySqlStreamStoreDb.cs b/src/LoadTests/MySqlStreamStoreDb.cs deleted file mode 100644 index c5822d33a..000000000 --- a/src/LoadTests/MySqlStreamStoreDb.cs +++ /dev/null @@ -1,49 +0,0 @@ -namespace LoadTests -{ - using System; - using System.Threading.Tasks; - using SqlStreamStore; - using SqlStreamStore.TestUtils.MySql; - using Xunit.Abstractions; - - public class MySqlStreamStoreDb : IDisposable - { - private string ConnectionString => _databaseManager.ConnectionString; - private readonly MySqlContainer _databaseManager; - - public MySqlStreamStoreDb() - : this(new ConsoleTestoutputHelper()) - { - - } - public MySqlStreamStoreDb(ITestOutputHelper testOutputHelper) - { - _databaseManager = new MySqlContainer($"test_{Guid.NewGuid():n}"); - } - - public async Task GetMySqlStreamStore() - { - await CreateDatabase(); - - var settings = new MySqlStreamStoreSettings(ConnectionString); - - var mySqlStreamStore = new MySqlStreamStore(settings); - await mySqlStreamStore.CreateSchemaIfNotExists(); - return mySqlStreamStore; - } - - public Task CreateDatabase() => _databaseManager.CreateDatabase(); - - private class ConsoleTestoutputHelper : ITestOutputHelper - { - public void WriteLine(string message) => Console.Write(message); - - public void WriteLine(string format, params object[] args) => Console.WriteLine(format, args); - } - - public void Dispose() - { - - } - } -} \ No newline at end of file diff --git a/src/LoadTests/SqlLocalDb.cs b/src/LoadTests/SqlLocalDb.cs deleted file mode 100644 index 2b0d3b51c..000000000 --- a/src/LoadTests/SqlLocalDb.cs +++ /dev/null @@ -1,62 +0,0 @@ -namespace LoadTests -{ - using System; - using System.Threading.Tasks; - using MartinCostello.SqlLocalDb; - using Microsoft.Data.SqlClient; - using SqlStreamStore; - - public class SqlLocalDb : IDisposable - { - private readonly ISqlLocalDbInstanceInfo _instance; - private readonly ISqlLocalDbInstanceManager _manager; - private MsSqlStreamStoreV3 _msSqlStreamStoreV3; - private readonly SqlLocalDbApi _localDb; - private readonly string _databaseName = Guid.NewGuid().ToString(); - - public SqlLocalDb() - { - _localDb = new SqlLocalDbApi(); - _instance = _localDb.GetOrCreateInstance("SSS-LoadTests"); - _manager = _instance.Manage(); - Initialize().Wait(); - } - - private async Task Initialize() - { - if (!_instance.IsRunning) - { - _manager.Start(); - } - - using (var connection = _instance.CreateConnection()) - { - await connection.OpenAsync(); - var tempPath = Environment.GetEnvironmentVariable("Temp"); - var createDatabase = $"CREATE DATABASE [{_databaseName}] on (name='{_databaseName}', " - + $"filename='{tempPath}\\{_databaseName}.mdf')"; - using (var command = new SqlCommand(createDatabase, connection)) - { - await command.ExecuteNonQueryAsync(); - } - } - - var sqlConnectionStringBuilder = _instance.CreateConnectionStringBuilder(); - sqlConnectionStringBuilder.InitialCatalog = _databaseName; - ConnectionString = sqlConnectionStringBuilder.ToString(); - var settings = new MsSqlStreamStoreV3Settings(ConnectionString); - _msSqlStreamStoreV3 = new MsSqlStreamStoreV3(settings); - await _msSqlStreamStoreV3.CreateSchemaIfNotExists(); - } - - public IStreamStore StreamStore => _msSqlStreamStoreV3; - - public string ConnectionString { get; private set; } - - public void Dispose() - { - _msSqlStreamStoreV3.Dispose(); - _localDb.Dispose(); - } - } -} \ No newline at end of file diff --git a/src/LoadTests/TestGaps.cs b/src/LoadTests/TestGaps.cs index d686ddebd..ee673fb96 100644 --- a/src/LoadTests/TestGaps.cs +++ b/src/LoadTests/TestGaps.cs @@ -28,8 +28,7 @@ public override async Task Run(CancellationToken ct) int readPageSize = Input.ReadInt("Read page size: ", 1, 10000); - string jsonData = new string('a', messageJsonDataSize * 1024); - + string jsonData = $@"{{""b"": ""{new string('a', messageJsonDataSize * 1024)}""}}"; // new string('a', messageJsonDataSize * 1024); var linkedToken = CancellationTokenSource.CreateLinkedTokenSource(ct); for(int i = 0; i < 10; i++) diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/AppendToStream.sql b/src/SqlStreamStore.Postgres/PgSqlScripts/AppendToStream.sql index d54a1a366..7499a2601 100644 --- a/src/SqlStreamStore.Postgres/PgSqlScripts/AppendToStream.sql +++ b/src/SqlStreamStore.Postgres/PgSqlScripts/AppendToStream.sql @@ -105,9 +105,10 @@ BEGIN created_utc, type, json_data, - json_metadata) + json_metadata, + tx_id) SELECT m.message_id, _stream_id_internal, _current_version + (row_number() - over ()) :: int, _created_utc, m.type, m.json_data, m.json_metadata + over ()) :: int, _created_utc, m.type, m.json_data, m.json_metadata, txid_current() FROM unnest(_new_stream_messages) m ON CONFLICT DO NOTHING; GET DIAGNOSTICS _success = ROW_COUNT; diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/Read.sql b/src/SqlStreamStore.Postgres/PgSqlScripts/Read.sql index 3dee4eb07..f8a1e76a9 100644 --- a/src/SqlStreamStore.Postgres/PgSqlScripts/Read.sql +++ b/src/SqlStreamStore.Postgres/PgSqlScripts/Read.sql @@ -42,6 +42,7 @@ BEGIN WHERE (CASE WHEN _forwards THEN __schema__.messages.stream_version >= _version AND id_internal = _stream_id_internal ELSE __schema__.messages.stream_version <= _version AND id_internal = _stream_id_internal END) + AND __schema__.messages.tx_id < txid_snapshot_xmin(txid_current_snapshot()) ORDER BY (CASE WHEN _forwards THEN __schema__.messages.stream_version ELSE -__schema__.messages.stream_version END) diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/ReadAll.sql b/src/SqlStreamStore.Postgres/PgSqlScripts/ReadAll.sql index c036e245a..236c7e54b 100644 --- a/src/SqlStreamStore.Postgres/PgSqlScripts/ReadAll.sql +++ b/src/SqlStreamStore.Postgres/PgSqlScripts/ReadAll.sql @@ -1,9 +1,9 @@ CREATE OR REPLACE FUNCTION __schema__.read_all2( _count INT, _position BIGINT, - _max_position BIGINT, _forwards BOOLEAN, - _prefetch BOOLEAN + _prefetch BOOLEAN, + _max_position BIGINT ) -- RETURNS TABLE( -- stream_id VARCHAR(1000), @@ -44,9 +44,10 @@ BEGIN FROM __schema__.messages INNER JOIN __schema__.streams ON __schema__.messages.stream_id_internal = __schema__.streams.id_internal WHERE (CASE - WHEN _forwards && _max_position > 0 THEN __schema__.messages.position >= _position AND __schema__.messages.position <= _max_position + WHEN _forwards AND _max_position > 0 THEN __schema__.messages.position >= _position AND __schema__.messages.position <= _max_position WHEN _forwards THEN __schema__.messages.position >= _position ELSE __schema__.messages.position <= _position END) + AND __schema__.messages.tx_id < txid_snapshot_xmin(txid_current_snapshot()) ORDER BY (CASE WHEN _forwards THEN __schema__.messages.position END), (CASE WHEN not _forwards THEN __schema__.messages.position END) DESC diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/Tables.sql b/src/SqlStreamStore.Postgres/PgSqlScripts/Tables.sql index 041df2ea8..e7eb520cf 100644 --- a/src/SqlStreamStore.Postgres/PgSqlScripts/Tables.sql +++ b/src/SqlStreamStore.Postgres/PgSqlScripts/Tables.sql @@ -37,8 +37,9 @@ CREATE TABLE IF NOT EXISTS __schema__.messages ( message_id UUID NOT NULL, created_utc TIMESTAMP NOT NULL, type VARCHAR(128) NOT NULL, - json_data JSONB NOT NULL, + json_data JSONB NOT NULL, json_metadata JSONB, + tx_id BIGINT NOT NULL DEFAULT 0, CONSTRAINT pk_messages PRIMARY KEY (position), CONSTRAINT fk_messages_stream FOREIGN KEY (stream_id_internal) REFERENCES __schema__.streams (id_internal), CONSTRAINT uq_messages_stream_id_internal_and_stream_version UNIQUE (stream_id_internal, stream_version), diff --git a/src/SqlStreamStore.Postgres/PostgresStreamStore.ReadAll.cs b/src/SqlStreamStore.Postgres/PostgresStreamStore.ReadAll.cs index 0e50fe6d6..f67b89215 100644 --- a/src/SqlStreamStore.Postgres/PostgresStreamStore.ReadAll.cs +++ b/src/SqlStreamStore.Postgres/PostgresStreamStore.ReadAll.cs @@ -20,7 +20,8 @@ protected override async Task ReadAllForwardsInternal( int maxCount, bool prefetch, ReadNextAllPage readNext, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + long fromMaxPositionInclusive = -1) { maxCount = maxCount == int.MaxValue ? maxCount - 1 : maxCount; @@ -35,7 +36,8 @@ protected override async Task ReadAllForwardsInternal( Parameters.Count(maxCount + 1), Parameters.Position(fromPositionExclusive), Parameters.ReadDirection(ReadDirection.Forward), - Parameters.Prefetch(prefetch))) + Parameters.Prefetch(prefetch), + Parameters.Position(fromMaxPositionInclusive))) using(var reader = await command .ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken) .ConfigureAwait(false)) diff --git a/src/SqlStreamStore.TestUtils/Postgres/PostgresContainer.cs b/src/SqlStreamStore.TestUtils/Postgres/PostgresContainer.cs index 1d93955bf..bc7969324 100644 --- a/src/SqlStreamStore.TestUtils/Postgres/PostgresContainer.cs +++ b/src/SqlStreamStore.TestUtils/Postgres/PostgresContainer.cs @@ -57,7 +57,8 @@ await Policy Username = "postgres", Host = "localhost", Pooling = true, - MaxPoolSize = 1024 + MaxPoolSize = 1024, + IncludeErrorDetails = true }; } } diff --git a/src/SqlStreamStore/InMemory/InMemoryAllStream.cs b/src/SqlStreamStore/InMemory/InMemoryAllStream.cs deleted file mode 100644 index cf255ab87..000000000 --- a/src/SqlStreamStore/InMemory/InMemoryAllStream.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace SqlStreamStore.InMemory -{ - using System.Collections.Generic; - - internal class InMemoryAllStream : LinkedList - {} -} \ No newline at end of file diff --git a/src/SqlStreamStore/InMemory/InMemoryStream.cs b/src/SqlStreamStore/InMemory/InMemoryStream.cs deleted file mode 100644 index db9e659d3..000000000 --- a/src/SqlStreamStore/InMemory/InMemoryStream.cs +++ /dev/null @@ -1,225 +0,0 @@ -namespace SqlStreamStore.InMemory -{ - using System; - using System.Collections.Generic; - using System.Linq; - using SqlStreamStore.Infrastructure; - using SqlStreamStore.Streams; - - internal class InMemoryStream - { - private readonly string _streamId; - private readonly InMemoryAllStream _inMemoryAllStream; - private readonly GetUtcNow _getUtcNow; - private readonly Action _onStreamAppended; - private readonly Func _getNextPosition; - private readonly List _messages = new List(); - private readonly Dictionary _messagesById = new Dictionary(); - - internal InMemoryStream( - string streamId, - InMemoryAllStream inMemoryAllStream, - GetUtcNow getUtcNow, - Action onStreamAppended, - Func getNextPosition) - { - _streamId = streamId; - _inMemoryAllStream = inMemoryAllStream; - _getUtcNow = getUtcNow; - _onStreamAppended = onStreamAppended; - _getNextPosition = getNextPosition; - } - - internal IReadOnlyList Messages => _messages; - - internal int CurrentVersion { get; private set; } = -1; - - internal int CurrentPosition { get; private set; } = -1; - - internal void AppendToStream(int expectedVersion, NewStreamMessage[] newMessages) - { - switch(expectedVersion) - { - case ExpectedVersion.Any: - AppendToStreamExpectedVersionAny(expectedVersion, newMessages); - return; - case ExpectedVersion.NoStream: - AppendToStreamExpectedVersionNoStream(expectedVersion, newMessages); - return; - default: - AppendToStreamExpectedVersion(expectedVersion, newMessages); - return; - } - } - - private void AppendToStreamExpectedVersion(int expectedVersion, NewStreamMessage[] newMessages) - { - // Need to do optimistic concurrency check... - if(expectedVersion > CurrentVersion) - { - throw new WrongExpectedVersionException( - ErrorMessages.AppendFailedWrongExpectedVersion(_streamId, expectedVersion), - _streamId, - expectedVersion); - } - - if(CurrentVersion >= 0 && expectedVersion < CurrentVersion) - { - // expectedVersion < currentVersion, Idempotency test - for(int i = 0; i < newMessages.Length; i++) - { - int index = expectedVersion + i + 1; - if(index >= _messages.Count) - { - throw new WrongExpectedVersionException( - ErrorMessages.AppendFailedWrongExpectedVersion(_streamId, expectedVersion), - _streamId, - expectedVersion); - } - if(_messages[index].MessageId != newMessages[i].MessageId) - { - throw new WrongExpectedVersionException( - ErrorMessages.AppendFailedWrongExpectedVersion(_streamId, expectedVersion), - _streamId, - expectedVersion); - } - } - return; - } - - // expectedVersion == currentVersion) - if(newMessages.Any(newmessage => _messagesById.ContainsKey(newmessage.MessageId))) - { - throw new WrongExpectedVersionException( - ErrorMessages.AppendFailedWrongExpectedVersion(_streamId, expectedVersion), - _streamId, - expectedVersion); - } - - AppendEvents(newMessages); - } - - private void AppendToStreamExpectedVersionAny(int expectedVersion, NewStreamMessage[] newMessages) - { - if(newMessages?.Length > 0) - { - // idemponcy check - have messages already been written? - if(_messagesById.TryGetValue(newMessages[0].MessageId, out var item)) - { - int i = _messages.IndexOf(item); - if(i + newMessages.Length > _messages.Count) - { - throw new WrongExpectedVersionException( - ErrorMessages.AppendFailedWrongExpectedVersion(_streamId, expectedVersion), - _streamId, - expectedVersion); - } - - for(int n = 1; n < newMessages.Length; n++) - { - if(newMessages[n].MessageId != _messages[i + n].MessageId) - { - throw new WrongExpectedVersionException( - ErrorMessages.AppendFailedWrongExpectedVersion(_streamId, expectedVersion), - _streamId, - expectedVersion); - } - } - - return; - } - } - - // None of the Messages were written previously... - AppendEvents(newMessages); - } - - private void AppendToStreamExpectedVersionNoStream(int expectedVersion, NewStreamMessage[] newMessages) - { - if(_messages.Count > 0) - { - //Already committed Messages, do idempotency check - if(newMessages.Length > _messages.Count) - { - throw new WrongExpectedVersionException( - ErrorMessages.AppendFailedWrongExpectedVersion(_streamId, expectedVersion), - _streamId, - expectedVersion); - } - - if(newMessages.Where((message, index) => _messages[index].MessageId != message.MessageId).Any()) - { - throw new WrongExpectedVersionException( - ErrorMessages.AppendFailedWrongExpectedVersion(_streamId, expectedVersion), - _streamId, - expectedVersion); - } - return; - } - - // None of the Messages were written previously... - AppendEvents(newMessages); - } - - private void AppendEvents(NewStreamMessage[] newMessages) - { - foreach(var newmessage in newMessages) - { - var position = _getNextPosition(); - CurrentVersion++; - CurrentPosition = position; - - var inMemorymessage = new InMemoryStreamMessage( - _streamId, - newmessage.MessageId, - CurrentVersion, - position, - _getUtcNow(), - newmessage.Type, - newmessage.JsonData, - newmessage.JsonMetadata); - - _messages.Add(inMemorymessage); - _messagesById.Add(newmessage.MessageId, inMemorymessage); - _inMemoryAllStream.AddAfter(_inMemoryAllStream.Last, inMemorymessage); - } - _onStreamAppended(); - } - - internal void DeleteAllEvents(int expectedVersion) - { - if (expectedVersion > 0 && expectedVersion != CurrentVersion) - { - throw new WrongExpectedVersionException( - ErrorMessages.AppendFailedWrongExpectedVersion(_streamId, expectedVersion), - _streamId, - expectedVersion); - } - - foreach (var inMemorymessage in _messages) - { - _inMemoryAllStream.Remove(inMemorymessage); - } - _messages.Clear(); - _messagesById.Clear(); - } - - internal bool DeleteEvent(Guid eventId) - { - if (!_messagesById.TryGetValue(eventId, out var inMemoryStreamMessage)) - { - return false; - } - - _messages.Remove(inMemoryStreamMessage); - _inMemoryAllStream.Remove(inMemoryStreamMessage); - _messagesById.Remove(eventId); - return true; - } - - internal string GetMessageData(Guid messageId) - { - return _messagesById.TryGetValue(messageId, out var message) ? message.JsonData : string.Empty; - } - } -} \ No newline at end of file diff --git a/src/SqlStreamStore/InMemory/InMemoryStreamMessage.cs b/src/SqlStreamStore/InMemory/InMemoryStreamMessage.cs deleted file mode 100644 index 51c59d9c4..000000000 --- a/src/SqlStreamStore/InMemory/InMemoryStreamMessage.cs +++ /dev/null @@ -1,36 +0,0 @@ -namespace SqlStreamStore.InMemory -{ - using System; - - internal sealed class InMemoryStreamMessage - { - internal readonly long Position; - internal readonly DateTime Created; - internal readonly string StreamId; - internal readonly Guid MessageId; - internal readonly string JsonData; - internal readonly string JsonMetadata; - internal readonly int StreamVersion; - internal readonly string Type; - - internal InMemoryStreamMessage( - string streamId, - Guid messageId, - int streamVersion, - long position, - DateTime created, - string type, - string jsonData, - string jsonMetadata) - { - StreamId = streamId; - MessageId = messageId; - StreamVersion = streamVersion; - Position = position; - Created = created; - Type = type; - JsonData = jsonData; - JsonMetadata = jsonMetadata; - } - } -} \ No newline at end of file diff --git a/src/SqlStreamStore/InMemory/InMemoryStreamStore.cs b/src/SqlStreamStore/InMemory/InMemoryStreamStore.cs deleted file mode 100644 index aa9fd88ed..000000000 --- a/src/SqlStreamStore/InMemory/InMemoryStreamStore.cs +++ /dev/null @@ -1,781 +0,0 @@ -// ReSharper disable once CheckNamespace -namespace SqlStreamStore -{ - using System; - using System.Collections.Generic; - using System.Linq; - using System.Threading; - using System.Threading.Tasks; - using SqlStreamStore.Imports.Ensure.That; - using SqlStreamStore.Infrastructure; - using SqlStreamStore.InMemory; - using SqlStreamStore.Streams; - using SqlStreamStore.Subscriptions; - using StreamStoreStore.Json; - using static Streams.Deleted; - - /// - /// Represents an in-memory implementation of a stream store. Use for testing or high/speed + volatile scenarios. - /// - public sealed class InMemoryStreamStore : StreamStoreBase - { - private readonly InMemoryAllStream _allStream = new InMemoryAllStream(); - private readonly GetUtcNow _getUtcNow; - private readonly ReaderWriterLockSlim _lock = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion); - private readonly Action _onStreamAppended; - private readonly Dictionary _streams = new Dictionary(); - private readonly Subject _subscriptions = new Subject(); - private readonly InterlockedBoolean _signallingToSubscribers = new InterlockedBoolean(); - private readonly IList _streamIds = new List(); - private int _currentPosition; - private static readonly ReadNextStreamPage s_readNextNotFound = - (_, ct) => throw new InvalidOperationException("Cannot read next page of non-exisitent stream"); - - - /// - /// Initializes a new instance of - /// - /// Optional. A delegate to ge the current UTC now. Used to define generate timestamps. - /// Optional. The name of the logger generated log messages will use. - public InMemoryStreamStore(GetUtcNow getUtcNow = null, string logName = null) - : base(TimeSpan.FromMinutes(1), 10000, getUtcNow, logName ?? nameof(InMemoryStreamStore)) - { - _getUtcNow = getUtcNow ?? SystemClock.GetUtcNow; - _allStream.AddFirst(new InMemoryStreamMessage( - "", - Guid.NewGuid(), - -1, - -1, - _getUtcNow(), - null, - null, - null)); - _onStreamAppended = () => - { - if(_signallingToSubscribers.CompareExchange(true, false) == false) - { - Task.Run(() => - { - _subscriptions.OnNext(Unit.Default); - _signallingToSubscribers.Set(false); - }); - } - }; - } - - protected override void Dispose(bool disposing) - { - using(_lock.UseWriteLock()) - { - _subscriptions.OnCompleted(); - _allStream.Clear(); - _streams.Clear(); - } - } - - private Task GetStreamMessageCount(string streamId, CancellationToken cancellationToken) - { - using(_lock.UseReadLock()) - { - return Task.FromResult(!_streams.ContainsKey(streamId) ? 0 : _streams[streamId].Messages.Count); - } - } - - protected override async Task AppendToStreamInternal( - string streamId, - int expectedVersion, - NewStreamMessage[] messages, - CancellationToken cancellationToken) - { - GuardAgainstDisposed(); - cancellationToken.ThrowIfCancellationRequested(); - - AppendResult appendResult; - using (_lock.UseWriteLock()) - { - appendResult = AppendToStreamInternal(streamId, expectedVersion, messages); - } - var meta = await GetStreamMetadataInternal(streamId, cancellationToken); - await CheckStreamMaxCount(streamId, meta.MaxCount, cancellationToken); - return appendResult; - } - - private async Task CheckStreamMaxCount(string streamId, int? maxCount, CancellationToken cancellationToken) - { - if (maxCount.HasValue) - { - var count = await GetStreamMessageCount(streamId, cancellationToken); - if (count > maxCount.Value) - { - int toPurge = count - maxCount.Value; - - var streamMessagesPage = await ReadStreamForwardsInternal(streamId, StreamVersion.Start, - toPurge, false, null, cancellationToken); - - if (streamMessagesPage.Status == PageReadStatus.Success) - { - foreach (var message in streamMessagesPage.Messages) - { - await DeleteEventInternal(streamId, message.MessageId, cancellationToken); - } - } - } - } - } - - private AppendResult AppendToStreamInternal( - string streamId, - int expectedVersion, - NewStreamMessage[] messages) - { - InMemoryStream inMemoryStream; - if (expectedVersion == ExpectedVersion.NoStream || expectedVersion == ExpectedVersion.Any) - { - if (_streams.TryGetValue(streamId, out inMemoryStream)) - { - inMemoryStream.AppendToStream(expectedVersion, messages); - } - else - { - inMemoryStream = new InMemoryStream( - streamId, - _allStream, - _getUtcNow, - _onStreamAppended, - () => _currentPosition++); - inMemoryStream.AppendToStream(expectedVersion, messages); - _streams.Add(streamId, inMemoryStream); - _streamIds.Add(streamId); - } - return new AppendResult(inMemoryStream.CurrentVersion, inMemoryStream.CurrentPosition); - } - - if (!_streams.TryGetValue(streamId, out inMemoryStream)) - { - throw new WrongExpectedVersionException( - ErrorMessages.AppendFailedWrongExpectedVersion(streamId, expectedVersion), - streamId, - expectedVersion); - } - inMemoryStream.AppendToStream(expectedVersion, messages); - return new AppendResult(inMemoryStream.CurrentVersion, inMemoryStream.CurrentPosition); - } - - protected override Task DeleteEventInternal(string streamId, Guid eventId, CancellationToken cancellationToken) - { - GuardAgainstDisposed(); - cancellationToken.ThrowIfCancellationRequested(); - - using (_lock.UseWriteLock()) - { - if (!_streams.ContainsKey(streamId)) - { - return Task.FromResult(0); - } - - var inMemoryStream = _streams[streamId]; - bool deleted = inMemoryStream.DeleteEvent(eventId); - - if (deleted) - { - var eventDeletedEvent = CreateMessageDeletedMessage(streamId, eventId); - AppendToStreamInternal(DeletedStreamId, ExpectedVersion.Any, new[] { eventDeletedEvent }); - } - - return Task.FromResult(0); - } - } - - protected override async Task GetStreamMetadataInternal( - string streamId, - CancellationToken cancellationToken) - { - GuardAgainstDisposed(); - cancellationToken.ThrowIfCancellationRequested(); - - using (_lock.UseReadLock()) - { - string metaStreamId = $"$${streamId}"; - - var eventsPage = await ReadStreamBackwardsInternal(metaStreamId, StreamVersion.End, - 1, true, null, cancellationToken); - - if (eventsPage.Status == PageReadStatus.StreamNotFound) - { - return new StreamMetadataResult(streamId, -1); - } - - var metadataMessage = await eventsPage.Messages[0].GetJsonDataAs(cancellationToken); - - return new StreamMetadataResult( - streamId, - eventsPage.LastStreamVersion, - metadataMessage.MaxAge, - metadataMessage.MaxCount, - metadataMessage.MetaJson); - } - } - - protected override async Task SetStreamMetadataInternal( - string streamId, - int expectedStreamMetadataVersion, - int? maxAge, - int? maxCount, - string metadataJson, - CancellationToken cancellationToken) - { - using(_lock.UseWriteLock()) - { - string metaStreamId = $"$${streamId}"; - - var metadataMessage = new MetadataMessage - { - StreamId = streamId, - MaxAge = maxAge, - MaxCount = maxCount, - MetaJson = metadataJson - }; - var json = SimpleJson.SerializeObject(metadataMessage); - var messageId = MetadataMessageIdGenerator.Create(metaStreamId, expectedStreamMetadataVersion, json); - var newStreamMessage = new NewStreamMessage(messageId, "$stream-metadata", json); - - var result = AppendToStreamInternal(metaStreamId, expectedStreamMetadataVersion, new[] { newStreamMessage }); - - await CheckStreamMaxCount(streamId, metadataMessage.MaxCount, cancellationToken); - - return new SetStreamMetadataResult(result.CurrentVersion); - } - } - - protected override Task DeleteStreamInternal( - string streamId, - int expectedVersion, - CancellationToken cancellationToken) - { - GuardAgainstDisposed(); - - using(_lock.UseWriteLock()) - { - DeleteStream(streamId, expectedVersion); - - // Delete metadata stream, if it exists - DeleteStream($"$${streamId}", ExpectedVersion.Any); - - return Task.FromResult(0); - } - } - - private void DeleteStream(string streamId, int expectedVersion) - { - if (!_streams.ContainsKey(streamId)) - { - if (expectedVersion >= 0) - { - throw new WrongExpectedVersionException( - ErrorMessages.AppendFailedWrongExpectedVersion(streamId, expectedVersion), - streamId, - expectedVersion); - } - return; - } - if (expectedVersion != ExpectedVersion.Any && - _streams[streamId].Messages.Last().StreamVersion != expectedVersion) - { - throw new WrongExpectedVersionException( - ErrorMessages.AppendFailedWrongExpectedVersion(streamId, expectedVersion), - streamId, - expectedVersion); - } - InMemoryStream inMemoryStream = _streams[streamId]; - _streams.Remove(streamId); - inMemoryStream.DeleteAllEvents(ExpectedVersion.Any); - _streamIds[_streamIds.IndexOf(streamId)] = null; - - var streamDeletedEvent = CreateStreamDeletedMessage(streamId); - AppendToStreamInternal(DeletedStreamId, ExpectedVersion.Any, new[] { streamDeletedEvent }); - } - - protected override Task ReadAllForwardsInternal(long fromPositionExclusive, int maxCount, - bool prefetch, ReadNextAllPage readNext, CancellationToken cancellationToken) - { - GuardAgainstDisposed(); - cancellationToken.ThrowIfCancellationRequested(); - - using (_lock.UseReadLock()) - { - // Find the node to start from (it may not be equal to the exact position) - var current = _allStream.First; - if(current.Next == null) //Empty store - { - var result = new ReadAllPage(Position.Start, Position.Start, true, ReadDirection.Forward, readNext); - return Task.FromResult(result); - } - - var previous = current.Previous; - while(current.Value.Position < fromPositionExclusive) - { - if(current.Next == null) // fromPosition is past end of store - { - var result = new ReadAllPage(fromPositionExclusive, fromPositionExclusive, true, - ReadDirection.Forward, readNext); - return Task.FromResult(result); - } - previous = current; - current = current.Next; - } - - var messages = new List(); - while(maxCount > 0 && current != null) - { - StreamMessage message; - if (prefetch) - { - message = new StreamMessage( - current.Value.StreamId, - current.Value.MessageId, - current.Value.StreamVersion, - current.Value.Position, - current.Value.Created, - current.Value.Type, - current.Value.JsonMetadata, - current.Value.JsonData); - } - else - { - var currentCopy = current; - message = new StreamMessage( - current.Value.StreamId, - current.Value.MessageId, - current.Value.StreamVersion, - current.Value.Position, - current.Value.Created, - current.Value.Type, - current.Value.JsonMetadata, - ct => - { - return Task.Run( - () => ReadMessageData(currentCopy.Value.StreamId, currentCopy.Value.MessageId), ct); - }); - } - messages.Add(message); - maxCount--; - previous = current; - current = current.Next; - } - - var isEnd = current == null; - var nextPosition = current?.Value.Position ?? previous.Value.Position + 1; - fromPositionExclusive = messages.Any() ? messages[0].Position : 0; - - var page = new ReadAllPage( - fromPositionExclusive, - nextPosition, - isEnd, - ReadDirection.Forward, - readNext, - messages.ToArray()); - - return Task.FromResult(page); - } - } - - protected override Task ReadAllBackwardsInternal( - long fromPositionExclusive, - int maxCount, - bool prefetch, - ReadNextAllPage readNext, - CancellationToken cancellationToken) - { - GuardAgainstDisposed(); - - using (_lock.UseReadLock()) - { - if (fromPositionExclusive == Position.End) - { - fromPositionExclusive = _allStream.Last.Value.Position; - } - - // Find the node to start from (it may not be equal to the exact position) - var current = _allStream.First; - if(current.Next == null) //Empty store - { - var result = new ReadAllPage(Position.Start, Position.Start, true, ReadDirection.Backward, readNext); - return Task.FromResult(result); - } - - var previous = current.Previous; - while(current.Value.Position < fromPositionExclusive) - { - if(current.Next == null) // fromPosition is past end of store - { - var result = new ReadAllPage(fromPositionExclusive, fromPositionExclusive, true, - ReadDirection.Backward, readNext); - return Task.FromResult(result); - } - previous = current; - current = current.Next; - } - - var messages = new List(); - while(maxCount > 0 && current != _allStream.First) - { - StreamMessage message; - if (prefetch) - { - message = new StreamMessage( - current.Value.StreamId, - current.Value.MessageId, - current.Value.StreamVersion, - current.Value.Position, - current.Value.Created, - current.Value.Type, - current.Value.JsonMetadata, - current.Value.JsonData); - } - else - { - var currentCopy = current; - message = new StreamMessage( - current.Value.StreamId, - current.Value.MessageId, - current.Value.StreamVersion, - current.Value.Position, - current.Value.Created, - current.Value.Type, - current.Value.JsonMetadata, - ct => - { - return Task.Run( - () => ReadMessageData(currentCopy.Value.StreamId, currentCopy.Value.MessageId), ct); - }); - } - messages.Add(message); - - maxCount--; - previous = current; - current = current.Previous; - } - - bool isEnd; - if(previous == null || previous.Value.Position == 0) - { - isEnd = true; - } - else - { - isEnd = false; - } - var nextPosition = isEnd - ? 0 - : current.Value.Position; - - fromPositionExclusive = messages.Any() ? messages[0].Position : 0; - - var page = new ReadAllPage( - fromPositionExclusive, - nextPosition, - isEnd, - ReadDirection.Backward, - readNext, - messages.ToArray()); - - return Task.FromResult(page); - } - } - - protected override Task ReadStreamForwardsInternal(string streamId, int start, int count, bool prefetch, ReadNextStreamPage readNext, CancellationToken cancellationToken) - { - GuardAgainstDisposed(); - cancellationToken.ThrowIfCancellationRequested(); - - using (_lock.UseReadLock()) - { - if(!_streams.TryGetValue(streamId, out var stream)) - { - var notFound = new ReadStreamPage( - streamId, - PageReadStatus.StreamNotFound, - start, - -1, - -1, - -1, - ReadDirection.Forward, - true, - s_readNextNotFound); - return Task.FromResult(notFound); - } - - var messages = new List(); - var i = start; - - while(i < stream.Messages.Count && count > 0) - { - var inMemorymessage = stream.Messages[i]; - StreamMessage message; - if (prefetch) - { - message = new StreamMessage( - streamId, - inMemorymessage.MessageId, - inMemorymessage.StreamVersion, - inMemorymessage.Position, - inMemorymessage.Created, - inMemorymessage.Type, - inMemorymessage.JsonMetadata, - inMemorymessage.JsonData); - } - else - { - message = new StreamMessage( - streamId, - inMemorymessage.MessageId, - inMemorymessage.StreamVersion, - inMemorymessage.Position, - inMemorymessage.Created, - inMemorymessage.Type, - inMemorymessage.JsonMetadata, - ct => Task.Run(() => ReadMessageData(streamId, inMemorymessage.MessageId), ct)); - } - messages.Add(message); - - i++; - count--; - } - - var lastStreamVersion = stream.CurrentVersion; - int nextStreamVersion; - if(lastStreamVersion == -1) - { - nextStreamVersion = 0; - } - else if(messages.Count == 0) - { - nextStreamVersion = lastStreamVersion + 1; - } - else - { - nextStreamVersion = messages.Last().StreamVersion + 1; - } - var endOfStream = i == stream.Messages.Count; - - var page = new ReadStreamPage( - streamId, - PageReadStatus.Success, - start, - nextStreamVersion, - lastStreamVersion, - stream.CurrentPosition, - ReadDirection.Forward, - endOfStream, - readNext, - messages.ToArray()); - - return Task.FromResult(page); - } - } - - protected override Task ReadStreamBackwardsInternal( - string streamId, - int fromVersionInclusive, - int count, - bool prefetch, - ReadNextStreamPage readNext, - CancellationToken cancellationToken) - { - GuardAgainstDisposed(); - cancellationToken.ThrowIfCancellationRequested(); - - using (_lock.UseReadLock()) - { - if (!_streams.TryGetValue(streamId, out var stream)) - { - var notFound = new ReadStreamPage(streamId, - PageReadStatus.StreamNotFound, - fromVersionInclusive, - -1, - -1, - -1, - ReadDirection.Backward, - true, - s_readNextNotFound); - return Task.FromResult(notFound); - } - - var messages = new List(); - var i = fromVersionInclusive == StreamVersion.End ? stream.Messages.Count - 1 : fromVersionInclusive; - while (i < stream.Messages.Count && i >= 0 && count > 0) - { - var inMemorymessage = stream.Messages[i]; - StreamMessage message; - if (prefetch) - { - message = new StreamMessage( - streamId, - inMemorymessage.MessageId, - inMemorymessage.StreamVersion, - inMemorymessage.Position, - inMemorymessage.Created, - inMemorymessage.Type, - inMemorymessage.JsonMetadata, - inMemorymessage.JsonData); - } - else - { - message = new StreamMessage( - streamId, - inMemorymessage.MessageId, - inMemorymessage.StreamVersion, - inMemorymessage.Position, - inMemorymessage.Created, - inMemorymessage.Type, - inMemorymessage.JsonMetadata, - ct => Task.Run(() => ReadMessageData(streamId, inMemorymessage.MessageId), ct)); - } - messages.Add(message); - - i--; - count--; - } - - var lastStreamVersion = stream.Messages.Count > 0 - ? stream.Messages[stream.Messages.Count - 1].StreamVersion - : StreamVersion.End; - var nextStreamVersion = messages.Count > 0 - ? messages[messages.Count - 1].StreamVersion - 1 - : StreamVersion.End; - var endOfStream = nextStreamVersion < 0; - - var page = new ReadStreamPage( - streamId, - PageReadStatus.Success, - fromVersionInclusive, - nextStreamVersion, - lastStreamVersion, - stream.CurrentPosition, - ReadDirection.Backward, - endOfStream, - readNext, - messages.ToArray()); - - return Task.FromResult(page); - } - } - - private string ReadMessageData(string streamId, Guid messageId) - { - using(_lock.UseReadLock()) - { - InMemoryStream stream; - return !_streams.TryGetValue(streamId, out stream) ? null : stream.GetMessageData(messageId); - } - } - - protected override IStreamSubscription SubscribeToStreamInternal( - string streamId, - int? startVersion, - StreamMessageReceived streamMessageReceived, - SubscriptionDropped subscriptionDropped, - HasCaughtUp hasCaughtUp, - bool prefetchJsonData, - string name) - { - return new StreamSubscription( - streamId, - startVersion, - this, - _subscriptions, - streamMessageReceived, - subscriptionDropped, - hasCaughtUp, - prefetchJsonData, - name); - } - - protected override Task ReadHeadPositionInternal(CancellationToken cancellationToken) - { - var message = _allStream.LastOrDefault(); - return message == null ? Task.FromResult(-1L) : Task.FromResult(message.Position); - } - - protected override Task ReadStreamHeadPositionInternal(string streamId, CancellationToken cancellationToken) - { - using(_lock.UseReadLock()) - { - return !_streams.TryGetValue(streamId, out InMemoryStream stream) ? Task.FromResult(-1L) : Task.FromResult(Convert.ToInt64(stream.CurrentPosition)); - } - } - - protected override Task ReadStreamHeadVersionInternal(string streamId, CancellationToken cancellationToken) - { - using(_lock.UseReadLock()) - { - return !_streams.TryGetValue(streamId, out InMemoryStream stream) ? Task.FromResult(-1) : Task.FromResult(stream.CurrentVersion); - } - } - - protected override IAllStreamSubscription SubscribeToAllInternal( - long? fromPosition, - AllStreamMessageReceived streamMessageReceived, - AllSubscriptionDropped subscriptionDropped, - HasCaughtUp hasCaughtUp, - bool prefetchJsonData, - string name) - { - return new AllStreamSubscription( - fromPosition, - this, - _subscriptions, - streamMessageReceived, - subscriptionDropped, - hasCaughtUp, - prefetchJsonData, - name); - } - - protected override Task ListStreamsInternal( - Pattern pattern, - int maxCount, - string continuationToken, - ListNextStreamsPage listNextStreamsPage, - CancellationToken cancellationToken) - { - Ensure.That(listNextStreamsPage).IsNotNull(); - int.TryParse(continuationToken, out var index); - - Func filter; - - switch(pattern) - { - case Pattern.Any _: - filter = s => s != default; - break; - case Pattern.StartingWith p: - filter = s => s != default && s.StartsWith(p.Value); - break; - case Pattern.EndingWith p: - filter = s => s != default && s.EndsWith(p.Value); - break; - default: - throw Pattern.Unrecognized(nameof(pattern)); - } - - using(_lock.UseReadLock()) - { - var streamIds = _streamIds - .Skip(index) - .Where(filter) - .Take(maxCount) - .ToArray(); - - var nextContinuationToken = streamIds.Length == 0 - ? 0 - : _streamIds.IndexOf(streamIds[streamIds.Length - 1]) + 1; - - return Task.FromResult(new ListStreamsPage( - nextContinuationToken.ToString(), - streamIds, - listNextStreamsPage)); - } - } - } -} \ No newline at end of file diff --git a/src/SqlStreamStore/Infrastructure/ReadonlyStreamStoreBase.cs b/src/SqlStreamStore/Infrastructure/ReadonlyStreamStoreBase.cs index a692ccdde..2e7cf92d8 100644 --- a/src/SqlStreamStore/Infrastructure/ReadonlyStreamStoreBase.cs +++ b/src/SqlStreamStore/Infrastructure/ReadonlyStreamStoreBase.cs @@ -396,8 +396,8 @@ private async Task ReloadAfterDelay( ReadNextAllPage readNext, CancellationToken cancellationToken) { - Logger.Info("ReadAllForwards: gap detected in position, reloading after {DefaultReloadInterval}ms", DefaultReloadInterval); - await Task.Delay(DefaultReloadInterval, cancellationToken); + Logger.Info("ReadAllForwards: gap detected in position, reloading after {DefaultReloadInterval}ms, position: {fromPositionInclusive}", DefaultReloadInterval, fromPositionInclusive); + //await Task.Delay(DefaultReloadInterval, cancellationToken); var reloadedPage = await ReadAllForwardsInternal(fromPositionInclusive, maxCount, prefetch, readNext, cancellationToken) .ConfigureAwait(false); return await FilterExpired(reloadedPage, readNext, cancellationToken).ConfigureAwait(false); diff --git a/src/SqlStreamStore/Streams/ReadAllPage.cs b/src/SqlStreamStore/Streams/ReadAllPage.cs index 4e20997c4..ae9ffb62f 100644 --- a/src/SqlStreamStore/Streams/ReadAllPage.cs +++ b/src/SqlStreamStore/Streams/ReadAllPage.cs @@ -64,7 +64,7 @@ public ReadAllPage( Direction = direction; _readNext = readNext; Messages = messages ?? new StreamMessage[0]; - TxSnapshot = new TxSnapshot(txSnapshot); + TxSnapshot = txSnapshot == null ? null : new TxSnapshot(txSnapshot); } /// diff --git a/tests/SqlStreamStore.HAL.Tests/SqlStreamStore.HAL.Tests.csproj b/tests/SqlStreamStore.HAL.Tests/SqlStreamStore.HAL.Tests.csproj index 71272c22f..19822778d 100644 --- a/tests/SqlStreamStore.HAL.Tests/SqlStreamStore.HAL.Tests.csproj +++ b/tests/SqlStreamStore.HAL.Tests/SqlStreamStore.HAL.Tests.csproj @@ -1,6 +1,6 @@  - netcoreapp2.1 + netcoreapp3.1 SqlStreamStore.HAL.Tests SqlStreamStore.HAL.Tests From 1750a2c34a643f70a1149550d2d267f9e63c84eb Mon Sep 17 00:00:00 2001 From: Arne Schoonvliet Date: Wed, 2 Feb 2022 09:07:09 +0100 Subject: [PATCH 03/13] Wip --- .../DevServerStartup.cs | 5 +- .../SqlStreamStoreFactory.cs | 2 +- .../AllStream/AllStreamResource.cs | 4 +- .../AllStream/ReadAllStreamOperation.cs | 2 +- .../AllStreamMessageResource.cs | 4 +- .../ReadAllStreamMessageOperation.cs | 2 +- .../IStreamStoreOperation.cs | 3 +- .../SqlStreamStoreHalMiddleware.cs | 3 +- .../StreamBrowser/ListStreamsOperation.cs | 2 +- .../StreamBrowser/StreamBrowserResource.cs | 5 +- ...DeleteStreamMessageByMessageIdOperation.cs | 3 +- .../StreamMessage/StreamMessageResource.cs | 4 +- .../DeleteStreamMessageByVersionOperation.cs | 3 +- ...adStreamMessageByStreamVersionOperation.cs | 2 +- .../GetStreamMetadataOperation.cs | 2 +- .../SetStreamMetadataOperation.cs | 3 +- .../StreamMetadata/StreamMetadataResource.cs | 5 +- .../Streams/AppendStreamOperation.cs | 2 +- .../Streams/DeleteStreamOperation.cs | 3 +- .../Streams/ReadStreamOperation.cs | 2 +- .../Streams/StreamResource.cs | 4 +- .../HttpClientSqlStreamStore.Subscriptions.cs | 2 +- .../HttpClientSqlStreamStore.cs | 2 +- .../MsSqlStreamStore.ReadAll.cs | 7 +- .../MsSqlStreamStore.Subscriptions.cs | 3 +- src/SqlStreamStore.MsSql/MsSqlStreamStore.cs | 2 +- .../MsSqlStreamStoreV3.ReadAll.cs | 7 +- .../MsSqlStreamStoreV3.Subscriptions.cs | 3 +- .../MsSqlStreamStoreV3.cs | 2 +- .../MySqlStreamStore.ReadAll.cs | 7 +- .../MySqlStreamStore.Subscriptions.cs | 3 +- src/SqlStreamStore.MySql/MySqlStreamStore.cs | 3 +- .../PgSqlScripts/ReadAll.sql | 1 - .../PostgresStreamStore.ReadAll.cs | 201 +++++++++++------- .../PostgresStreamStore.Subscriptions.cs | 3 +- .../PostgresStreamStore.cs | 53 ++++- .../SqliteStreamStore.ReadAll.cs | 7 +- .../SqliteStreamStore.Subscriptions.cs | 3 +- .../SqliteStreamStore.cs | 2 +- src/SqlStreamStore/IReadonlyStreamStore.cs | 117 +++++----- src/SqlStreamStore/IStreamStore.cs | 5 + .../Infrastructure/ReadonlyStreamStoreBase.cs | 196 ++++++++++++----- .../Infrastructure/StreamStoreBase.cs | 2 +- src/SqlStreamStore/StreamStoreExtensions.cs | 18 +- src/SqlStreamStore/Streams/ReadAllPage.cs | 64 +++--- src/SqlStreamStore/Streams/ReadNextAllPage.cs | 3 +- .../Subscriptions/AllStreamSubscription.cs | 12 +- .../CreateStreamStoreNotifier.cs | 2 +- 48 files changed, 499 insertions(+), 296 deletions(-) diff --git a/src/SqlStreamStore.HAL.DevServer/DevServerStartup.cs b/src/SqlStreamStore.HAL.DevServer/DevServerStartup.cs index 561fe3b45..4aa36ffeb 100644 --- a/src/SqlStreamStore.HAL.DevServer/DevServerStartup.cs +++ b/src/SqlStreamStore.HAL.DevServer/DevServerStartup.cs @@ -6,6 +6,7 @@ using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.DependencyInjection; + using SqlStreamStore.Streams; using MidFunc = System.Func< Microsoft.AspNetCore.Http.HttpContext, System.Func, @@ -14,11 +15,11 @@ internal class DevServerStartup : IStartup { - private readonly IStreamStore _streamStore; + private readonly IStreamStore _streamStore; private readonly SqlStreamStoreMiddlewareOptions _options; public DevServerStartup( - IStreamStore streamStore, + IStreamStore streamStore, SqlStreamStoreMiddlewareOptions options) { _streamStore = streamStore; diff --git a/src/SqlStreamStore.HAL.DevServer/SqlStreamStoreFactory.cs b/src/SqlStreamStore.HAL.DevServer/SqlStreamStoreFactory.cs index 57beb5d84..0b0e3317e 100644 --- a/src/SqlStreamStore.HAL.DevServer/SqlStreamStoreFactory.cs +++ b/src/SqlStreamStore.HAL.DevServer/SqlStreamStoreFactory.cs @@ -30,7 +30,7 @@ private static readonly IDictionary s_factories [mssql] = CreateMssqlStreamStore }; - public static Task Create(CancellationToken cancellationToken = default) + public static Task> Create(CancellationToken cancellationToken = default) { var provider = Environment.GetEnvironmentVariable(SQLSTREAMSTORE_PROVIDER)?.ToLowerInvariant() ?? inmemory; diff --git a/src/SqlStreamStore.HAL/AllStream/AllStreamResource.cs b/src/SqlStreamStore.HAL/AllStream/AllStreamResource.cs index 8901e41ce..a98673a2d 100644 --- a/src/SqlStreamStore.HAL/AllStream/AllStreamResource.cs +++ b/src/SqlStreamStore.HAL/AllStream/AllStreamResource.cs @@ -9,12 +9,12 @@ internal class AllStreamResource : IResource { - private readonly IStreamStore _streamStore; + private readonly IStreamStore _streamStore; private readonly bool _useCanonicalUrls; public SchemaSet Schema { get; } - public AllStreamResource(IStreamStore streamStore, bool useCanonicalUrls) + public AllStreamResource(IStreamStore streamStore, bool useCanonicalUrls) { if(streamStore == null) throw new ArgumentNullException(nameof(streamStore)); diff --git a/src/SqlStreamStore.HAL/AllStream/ReadAllStreamOperation.cs b/src/SqlStreamStore.HAL/AllStream/ReadAllStreamOperation.cs index 77d946442..d3d1e0c7e 100644 --- a/src/SqlStreamStore.HAL/AllStream/ReadAllStreamOperation.cs +++ b/src/SqlStreamStore.HAL/AllStream/ReadAllStreamOperation.cs @@ -66,7 +66,7 @@ public ReadAllStreamOperation(HttpContext context) public bool IsUriCanonical { get; } public PathString Path { get; } - public Task Invoke(IStreamStore streamStore, CancellationToken ct) + public Task Invoke(IStreamStore streamStore, CancellationToken ct) => ReadDirection == Constants.ReadDirection.Forwards ? streamStore.ReadAllForwards(_fromPositionInclusive, _maxCount, EmbedPayload, ct) : streamStore.ReadAllBackwards(_fromPositionInclusive, _maxCount, EmbedPayload, ct); diff --git a/src/SqlStreamStore.HAL/AllStreamMessage/AllStreamMessageResource.cs b/src/SqlStreamStore.HAL/AllStreamMessage/AllStreamMessageResource.cs index 5b09dfdf7..98e24fb75 100644 --- a/src/SqlStreamStore.HAL/AllStreamMessage/AllStreamMessageResource.cs +++ b/src/SqlStreamStore.HAL/AllStreamMessage/AllStreamMessageResource.cs @@ -8,11 +8,11 @@ internal class AllStreamMessageResource : IResource { - private readonly IStreamStore _streamStore; + private readonly IStreamStore _streamStore; public SchemaSet Schema { get; } - public AllStreamMessageResource(IStreamStore streamStore) + public AllStreamMessageResource(IStreamStore streamStore) { if(streamStore == null) throw new ArgumentNullException(nameof(streamStore)); diff --git a/src/SqlStreamStore.HAL/AllStreamMessage/ReadAllStreamMessageOperation.cs b/src/SqlStreamStore.HAL/AllStreamMessage/ReadAllStreamMessageOperation.cs index b22d55ad0..9262a8816 100644 --- a/src/SqlStreamStore.HAL/AllStreamMessage/ReadAllStreamMessageOperation.cs +++ b/src/SqlStreamStore.HAL/AllStreamMessage/ReadAllStreamMessageOperation.cs @@ -18,7 +18,7 @@ public ReadAllStreamMessageOperation(HttpContext context) public long Position { get; } public PathString Path { get; } - public async Task Invoke(IStreamStore streamStore, CancellationToken ct) + public async Task Invoke(IStreamStore streamStore, CancellationToken ct) { var page = await streamStore.ReadAllForwards(Position, 1, true, ct); diff --git a/src/SqlStreamStore.HAL/IStreamStoreOperation.cs b/src/SqlStreamStore.HAL/IStreamStoreOperation.cs index 85f11763f..bfb4c9298 100644 --- a/src/SqlStreamStore.HAL/IStreamStoreOperation.cs +++ b/src/SqlStreamStore.HAL/IStreamStoreOperation.cs @@ -3,10 +3,11 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Http; + using SqlStreamStore.Streams; internal interface IStreamStoreOperation { PathString Path { get; } - Task Invoke(IStreamStore streamStore, CancellationToken cancellationToken); + Task Invoke(IStreamStore streamStore, CancellationToken cancellationToken); } } \ No newline at end of file diff --git a/src/SqlStreamStore.HAL/SqlStreamStoreHalMiddleware.cs b/src/SqlStreamStore.HAL/SqlStreamStoreHalMiddleware.cs index 8a5ecc4c0..45882e381 100644 --- a/src/SqlStreamStore.HAL/SqlStreamStoreHalMiddleware.cs +++ b/src/SqlStreamStore.HAL/SqlStreamStoreHalMiddleware.cs @@ -17,6 +17,7 @@ using SqlStreamStore.HAL.StreamMessage; using SqlStreamStore.HAL.StreamMetadata; using SqlStreamStore.HAL.Streams; + using SqlStreamStore.Streams; using MidFunc = System.Func< Microsoft.AspNetCore.Http.HttpContext, System.Func, @@ -52,7 +53,7 @@ public static class SqlStreamStoreHalMiddleware public static IApplicationBuilder UseSqlStreamStoreHal( this IApplicationBuilder builder, - IStreamStore streamStore, + IStreamStore streamStore, SqlStreamStoreMiddlewareOptions options = default) { if(builder == null) diff --git a/src/SqlStreamStore.HAL/StreamBrowser/ListStreamsOperation.cs b/src/SqlStreamStore.HAL/StreamBrowser/ListStreamsOperation.cs index 56447154d..97897e897 100644 --- a/src/SqlStreamStore.HAL/StreamBrowser/ListStreamsOperation.cs +++ b/src/SqlStreamStore.HAL/StreamBrowser/ListStreamsOperation.cs @@ -48,7 +48,7 @@ public ListStreamsOperation(HttpContext context) : 100; } - public Task Invoke(IStreamStore streamStore, CancellationToken cancellationToken) + public Task Invoke(IStreamStore streamStore, CancellationToken cancellationToken) => streamStore.ListStreams(Pattern, MaxCount, ContinuationToken, cancellationToken); } } \ No newline at end of file diff --git a/src/SqlStreamStore.HAL/StreamBrowser/StreamBrowserResource.cs b/src/SqlStreamStore.HAL/StreamBrowser/StreamBrowserResource.cs index bdbebdef9..7207ba369 100644 --- a/src/SqlStreamStore.HAL/StreamBrowser/StreamBrowserResource.cs +++ b/src/SqlStreamStore.HAL/StreamBrowser/StreamBrowserResource.cs @@ -5,14 +5,15 @@ namespace SqlStreamStore.HAL.StreamBrowser using System.Threading.Tasks; using Halcyon.HAL; using SqlStreamStore.HAL.StreamBrowser; + using SqlStreamStore.Streams; internal class StreamBrowserResource : IResource { - private readonly IStreamStore _streamStore; + private readonly IStreamStore _streamStore; public SchemaSet Schema { get; } - public StreamBrowserResource(IStreamStore streamStore) + public StreamBrowserResource(IStreamStore streamStore) { _streamStore = streamStore; Schema = new SchemaSet(); diff --git a/src/SqlStreamStore.HAL/StreamMessage/MessageId/DeleteStreamMessageByMessageIdOperation.cs b/src/SqlStreamStore.HAL/StreamMessage/MessageId/DeleteStreamMessageByMessageIdOperation.cs index 3d4ceda06..25696ed25 100644 --- a/src/SqlStreamStore.HAL/StreamMessage/MessageId/DeleteStreamMessageByMessageIdOperation.cs +++ b/src/SqlStreamStore.HAL/StreamMessage/MessageId/DeleteStreamMessageByMessageIdOperation.cs @@ -5,6 +5,7 @@ namespace SqlStreamStore.HAL.StreamMessage.MessageId using System.Threading.Tasks; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Routing; + using SqlStreamStore.Streams; internal class DeleteStreamMessageByMessageIdOperation : IStreamStoreOperation { @@ -20,7 +21,7 @@ public DeleteStreamMessageByMessageIdOperation(HttpContext context) public Guid MessageId { get; } public PathString Path { get; } - public async Task Invoke(IStreamStore streamStore, CancellationToken ct) + public async Task Invoke(IStreamStore streamStore, CancellationToken ct) { await streamStore.DeleteMessage(StreamId, MessageId, ct); diff --git a/src/SqlStreamStore.HAL/StreamMessage/StreamMessageResource.cs b/src/SqlStreamStore.HAL/StreamMessage/StreamMessageResource.cs index 090772d5d..edd44e801 100644 --- a/src/SqlStreamStore.HAL/StreamMessage/StreamMessageResource.cs +++ b/src/SqlStreamStore.HAL/StreamMessage/StreamMessageResource.cs @@ -10,10 +10,10 @@ namespace SqlStreamStore.HAL.StreamMessage internal class StreamMessageResource : IResource { - private readonly IStreamStore _streamStore; + private readonly IStreamStore _streamStore; public SchemaSet Schema { get; } - public StreamMessageResource(IStreamStore streamStore) + public StreamMessageResource(IStreamStore streamStore) { if(streamStore == null) throw new ArgumentNullException(nameof(streamStore)); diff --git a/src/SqlStreamStore.HAL/StreamMessage/Version/DeleteStreamMessageByVersionOperation.cs b/src/SqlStreamStore.HAL/StreamMessage/Version/DeleteStreamMessageByVersionOperation.cs index a36835a0c..e9424f756 100644 --- a/src/SqlStreamStore.HAL/StreamMessage/Version/DeleteStreamMessageByVersionOperation.cs +++ b/src/SqlStreamStore.HAL/StreamMessage/Version/DeleteStreamMessageByVersionOperation.cs @@ -5,6 +5,7 @@ namespace SqlStreamStore.HAL.StreamMessage.Version using System.Threading.Tasks; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Routing; + using SqlStreamStore.Streams; internal class DeleteStreamMessageByVersionOperation : IStreamStoreOperation { @@ -20,7 +21,7 @@ public DeleteStreamMessageByVersionOperation(HttpContext context) public int StreamVersion { get; } public PathString Path { get; } - public async Task Invoke(IStreamStore streamStore, CancellationToken ct) + public async Task Invoke(IStreamStore streamStore, CancellationToken ct) { var messageId = (await streamStore.ReadStreamBackwards( StreamId, diff --git a/src/SqlStreamStore.HAL/StreamMessage/Version/ReadStreamMessageByStreamVersionOperation.cs b/src/SqlStreamStore.HAL/StreamMessage/Version/ReadStreamMessageByStreamVersionOperation.cs index 6a809cd3e..8032e064d 100644 --- a/src/SqlStreamStore.HAL/StreamMessage/Version/ReadStreamMessageByStreamVersionOperation.cs +++ b/src/SqlStreamStore.HAL/StreamMessage/Version/ReadStreamMessageByStreamVersionOperation.cs @@ -23,7 +23,7 @@ public ReadStreamMessageByStreamVersionOperation(HttpContext context) public int StreamVersion { get; } public string StreamId { get; } - public async Task Invoke(IStreamStore streamStore, CancellationToken ct) + public async Task Invoke(IStreamStore streamStore, CancellationToken ct) => (await streamStore.ReadStreamBackwards(StreamId, StreamVersion, 1, true, ct)) .Messages.FirstOrDefault(message => StreamVersion == SqlStreamStore.Streams.StreamVersion.End || message.StreamVersion == StreamVersion); diff --git a/src/SqlStreamStore.HAL/StreamMetadata/GetStreamMetadataOperation.cs b/src/SqlStreamStore.HAL/StreamMetadata/GetStreamMetadataOperation.cs index 0b9df371d..1861550a8 100644 --- a/src/SqlStreamStore.HAL/StreamMetadata/GetStreamMetadataOperation.cs +++ b/src/SqlStreamStore.HAL/StreamMetadata/GetStreamMetadataOperation.cs @@ -17,7 +17,7 @@ public GetStreamMetadataOperation(HttpContext context) public string StreamId { get; } public PathString Path { get; } - public Task Invoke(IStreamStore streamStore, CancellationToken ct) + public Task Invoke(IStreamStore streamStore, CancellationToken ct) => streamStore.GetStreamMetadata(StreamId, ct); } } \ No newline at end of file diff --git a/src/SqlStreamStore.HAL/StreamMetadata/SetStreamMetadataOperation.cs b/src/SqlStreamStore.HAL/StreamMetadata/SetStreamMetadataOperation.cs index 83c1fde32..fa090e80a 100644 --- a/src/SqlStreamStore.HAL/StreamMetadata/SetStreamMetadataOperation.cs +++ b/src/SqlStreamStore.HAL/StreamMetadata/SetStreamMetadataOperation.cs @@ -7,6 +7,7 @@ namespace SqlStreamStore.HAL.StreamMetadata using Microsoft.AspNetCore.Routing; using Newtonsoft.Json; using Newtonsoft.Json.Linq; + using SqlStreamStore.Streams; internal class SetStreamMetadataOperation : IStreamStoreOperation { @@ -41,7 +42,7 @@ private SetStreamMetadataOperation(HttpContext context, JObject body) public int? MaxCount { get; } public int? MaxAge { get; } - public async Task Invoke(IStreamStore streamStore, CancellationToken ct) + public async Task Invoke(IStreamStore streamStore, CancellationToken ct) { await streamStore.SetStreamMetadata( StreamId, diff --git a/src/SqlStreamStore.HAL/StreamMetadata/StreamMetadataResource.cs b/src/SqlStreamStore.HAL/StreamMetadata/StreamMetadataResource.cs index 1e0af2fad..8c5148fd0 100644 --- a/src/SqlStreamStore.HAL/StreamMetadata/StreamMetadataResource.cs +++ b/src/SqlStreamStore.HAL/StreamMetadata/StreamMetadataResource.cs @@ -5,13 +5,14 @@ namespace SqlStreamStore.HAL.StreamMetadata using System.Threading.Tasks; using Halcyon.HAL; using Newtonsoft.Json.Linq; + using SqlStreamStore.Streams; internal class StreamMetadataResource : IResource { - private readonly IStreamStore _streamStore; + private readonly IStreamStore _streamStore; public SchemaSet Schema { get; } - public StreamMetadataResource(IStreamStore streamStore) + public StreamMetadataResource(IStreamStore streamStore) { if(streamStore == null) throw new ArgumentNullException(nameof(streamStore)); diff --git a/src/SqlStreamStore.HAL/Streams/AppendStreamOperation.cs b/src/SqlStreamStore.HAL/Streams/AppendStreamOperation.cs index edc98e1d6..7b6559d48 100644 --- a/src/SqlStreamStore.HAL/Streams/AppendStreamOperation.cs +++ b/src/SqlStreamStore.HAL/Streams/AppendStreamOperation.cs @@ -86,7 +86,7 @@ private static NewStreamMessageDto ParseNewStreamMessage(JToken newStreamMessage public NewStreamMessageDto[] NewStreamMessages { get; } public PathString Path { get; } - public Task Invoke(IStreamStore streamStore, CancellationToken ct) + public Task Invoke(IStreamStore streamStore, CancellationToken ct) => streamStore.AppendToStream( StreamId, ExpectedVersion, diff --git a/src/SqlStreamStore.HAL/Streams/DeleteStreamOperation.cs b/src/SqlStreamStore.HAL/Streams/DeleteStreamOperation.cs index 6d42769ac..ef07dcfb6 100644 --- a/src/SqlStreamStore.HAL/Streams/DeleteStreamOperation.cs +++ b/src/SqlStreamStore.HAL/Streams/DeleteStreamOperation.cs @@ -4,6 +4,7 @@ namespace SqlStreamStore.HAL.Streams using System.Threading.Tasks; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Routing; + using SqlStreamStore.Streams; internal class DeleteStreamOperation : IStreamStoreOperation { @@ -18,7 +19,7 @@ public DeleteStreamOperation(HttpContext context) public int ExpectedVersion { get; } public PathString Path { get; } - public async Task Invoke(IStreamStore streamStore, CancellationToken ct) + public async Task Invoke(IStreamStore streamStore, CancellationToken ct) { await streamStore.DeleteStream(StreamId, ExpectedVersion, ct); diff --git a/src/SqlStreamStore.HAL/Streams/ReadStreamOperation.cs b/src/SqlStreamStore.HAL/Streams/ReadStreamOperation.cs index ef2defca6..8f3189181 100644 --- a/src/SqlStreamStore.HAL/Streams/ReadStreamOperation.cs +++ b/src/SqlStreamStore.HAL/Streams/ReadStreamOperation.cs @@ -67,7 +67,7 @@ public ReadStreamOperation(HttpContext context) public string Self { get; } public bool IsUriCanonical { get; } - public Task Invoke(IStreamStore streamStore, CancellationToken ct) + public Task Invoke(IStreamStore streamStore, CancellationToken ct) => ReadDirection == Constants.ReadDirection.Forwards ? streamStore.ReadStreamForwards(StreamId, _fromVersionInclusive, _maxCount, EmbedPayload, ct) : streamStore.ReadStreamBackwards(StreamId, _fromVersionInclusive, _maxCount, EmbedPayload, ct); diff --git a/src/SqlStreamStore.HAL/Streams/StreamResource.cs b/src/SqlStreamStore.HAL/Streams/StreamResource.cs index 116e44bcc..3b8e6336e 100644 --- a/src/SqlStreamStore.HAL/Streams/StreamResource.cs +++ b/src/SqlStreamStore.HAL/Streams/StreamResource.cs @@ -9,12 +9,12 @@ namespace SqlStreamStore.HAL.Streams internal class StreamResource : IResource { - private readonly IStreamStore _streamStore; + private readonly IStreamStore _streamStore; private readonly string _relativePathToRoot; private readonly bool _useCanonicalUrls; public SchemaSet Schema { get; } - public StreamResource(IStreamStore streamStore, bool useCanonicalUrls) + public StreamResource(IStreamStore streamStore, bool useCanonicalUrls) { if(streamStore == null) throw new ArgumentNullException(nameof(streamStore)); diff --git a/src/SqlStreamStore.Http/HttpClientSqlStreamStore.Subscriptions.cs b/src/SqlStreamStore.Http/HttpClientSqlStreamStore.Subscriptions.cs index 8c1f179e5..5ed90b6e8 100644 --- a/src/SqlStreamStore.Http/HttpClientSqlStreamStore.Subscriptions.cs +++ b/src/SqlStreamStore.Http/HttpClientSqlStreamStore.Subscriptions.cs @@ -38,7 +38,7 @@ public IAllStreamSubscription SubscribeToAll( { GuardAgainstDisposed(); - return new AllStreamSubscription( + return new AllStreamSubscription( continueAfterPosition, this, _streamStoreNotifier.Value, diff --git a/src/SqlStreamStore.Http/HttpClientSqlStreamStore.cs b/src/SqlStreamStore.Http/HttpClientSqlStreamStore.cs index 2c91b0649..5746ef484 100644 --- a/src/SqlStreamStore.Http/HttpClientSqlStreamStore.cs +++ b/src/SqlStreamStore.Http/HttpClientSqlStreamStore.cs @@ -14,7 +14,7 @@ using SqlStreamStore.Streams; using SqlStreamStore.Subscriptions; - public sealed partial class HttpClientSqlStreamStore : IStreamStore + public sealed partial class HttpClientSqlStreamStore : IStreamStore, IReadonlyStreamStore { private static readonly JsonSerializer s_serializer = JsonSerializer.Create(new JsonSerializerSettings { diff --git a/src/SqlStreamStore.MsSql/MsSqlStreamStore.ReadAll.cs b/src/SqlStreamStore.MsSql/MsSqlStreamStore.ReadAll.cs index 452765771..3ec7c478c 100644 --- a/src/SqlStreamStore.MsSql/MsSqlStreamStore.ReadAll.cs +++ b/src/SqlStreamStore.MsSql/MsSqlStreamStore.ReadAll.cs @@ -15,8 +15,9 @@ protected override async Task ReadAllForwardsInternal( long fromPosition, int maxCount, bool prefetch, - ReadNextAllPage readNext, - CancellationToken cancellationToken) + ReadNextAllPage readNext, + CancellationToken cancellationToken, + long fromMaxPositionInclusive = -1) { maxCount = maxCount == int.MaxValue ? maxCount - 1 : maxCount; long position = fromPosition; @@ -113,7 +114,7 @@ protected override async Task ReadAllBackwardsInternal( long fromPositionExclusive, int maxCount, bool prefetch, - ReadNextAllPage readNext, + ReadNextAllPage readNext, CancellationToken cancellationToken) { maxCount = maxCount == int.MaxValue ? maxCount - 1 : maxCount; diff --git a/src/SqlStreamStore.MsSql/MsSqlStreamStore.Subscriptions.cs b/src/SqlStreamStore.MsSql/MsSqlStreamStore.Subscriptions.cs index 93ddeadd4..d365f7baa 100644 --- a/src/SqlStreamStore.MsSql/MsSqlStreamStore.Subscriptions.cs +++ b/src/SqlStreamStore.MsSql/MsSqlStreamStore.Subscriptions.cs @@ -1,5 +1,6 @@ namespace SqlStreamStore { + using SqlStreamStore.Streams; using SqlStreamStore.Subscriptions; public sealed partial class MsSqlStreamStore @@ -33,7 +34,7 @@ protected override IAllStreamSubscription SubscribeToAllInternal( bool prefetchJsonData, string name) { - return new AllStreamSubscription( + return new AllStreamSubscription( fromPosition, this, GetStoreObservable, diff --git a/src/SqlStreamStore.MsSql/MsSqlStreamStore.cs b/src/SqlStreamStore.MsSql/MsSqlStreamStore.cs index c3a72dc26..3a77f6f86 100644 --- a/src/SqlStreamStore.MsSql/MsSqlStreamStore.cs +++ b/src/SqlStreamStore.MsSql/MsSqlStreamStore.cs @@ -17,7 +17,7 @@ /// Represents a Microsoft SQL Server stream store implementation. /// [Obsolete("Use MsSqlStreamStoreV3 instead. Note: this will require a schema and data migration.", false)] - public sealed partial class MsSqlStreamStore : StreamStoreBase + public sealed partial class MsSqlStreamStore : StreamStoreBase { private readonly Func _createConnection; private readonly Lazy _streamStoreNotifier; diff --git a/src/SqlStreamStore.MsSql/MsSqlStreamStoreV3.ReadAll.cs b/src/SqlStreamStore.MsSql/MsSqlStreamStoreV3.ReadAll.cs index a3c4d791f..0ac0b081c 100644 --- a/src/SqlStreamStore.MsSql/MsSqlStreamStoreV3.ReadAll.cs +++ b/src/SqlStreamStore.MsSql/MsSqlStreamStoreV3.ReadAll.cs @@ -15,8 +15,9 @@ protected override async Task ReadAllForwardsInternal( long fromPosition, int maxCount, bool prefetch, - ReadNextAllPage readNext, - CancellationToken cancellationToken) + ReadNextAllPage readNext, + CancellationToken cancellationToken, + long fromMaxPositionInclusive = -1) { maxCount = maxCount == int.MaxValue ? maxCount - 1 : maxCount; long position = fromPosition; @@ -111,7 +112,7 @@ protected override async Task ReadAllBackwardsInternal( long fromPositionExclusive, int maxCount, bool prefetch, - ReadNextAllPage readNext, + ReadNextAllPage readNext, CancellationToken cancellationToken) { maxCount = maxCount == int.MaxValue ? maxCount - 1 : maxCount; diff --git a/src/SqlStreamStore.MsSql/MsSqlStreamStoreV3.Subscriptions.cs b/src/SqlStreamStore.MsSql/MsSqlStreamStoreV3.Subscriptions.cs index 9e55704d4..0732d583e 100644 --- a/src/SqlStreamStore.MsSql/MsSqlStreamStoreV3.Subscriptions.cs +++ b/src/SqlStreamStore.MsSql/MsSqlStreamStoreV3.Subscriptions.cs @@ -1,5 +1,6 @@ namespace SqlStreamStore { + using SqlStreamStore.Streams; using SqlStreamStore.Subscriptions; public sealed partial class MsSqlStreamStoreV3 @@ -33,7 +34,7 @@ protected override IAllStreamSubscription SubscribeToAllInternal( bool prefetchJsonData, string name) { - return new AllStreamSubscription( + return new AllStreamSubscription( fromPosition, this, GetStoreObservable, diff --git a/src/SqlStreamStore.MsSql/MsSqlStreamStoreV3.cs b/src/SqlStreamStore.MsSql/MsSqlStreamStoreV3.cs index eb9da9c28..aad2e6c46 100644 --- a/src/SqlStreamStore.MsSql/MsSqlStreamStoreV3.cs +++ b/src/SqlStreamStore.MsSql/MsSqlStreamStoreV3.cs @@ -18,7 +18,7 @@ /// Represents a Microsoft SQL Server stream store implementation that /// uses V3 of the store schema. /// - public sealed partial class MsSqlStreamStoreV3 : StreamStoreBase + public sealed partial class MsSqlStreamStoreV3 : StreamStoreBase { private readonly Func _createConnection; private readonly Lazy _streamStoreNotifier; diff --git a/src/SqlStreamStore.MySql/MySqlStreamStore.ReadAll.cs b/src/SqlStreamStore.MySql/MySqlStreamStore.ReadAll.cs index d50ce2234..203c1134c 100644 --- a/src/SqlStreamStore.MySql/MySqlStreamStore.ReadAll.cs +++ b/src/SqlStreamStore.MySql/MySqlStreamStore.ReadAll.cs @@ -16,8 +16,9 @@ protected override async Task ReadAllForwardsInternal( long fromPositionExclusive, int maxCount, bool prefetch, - ReadNextAllPage readNext, - CancellationToken cancellationToken) + ReadNextAllPage readNext, + CancellationToken cancellationToken, + long fromMaxPositionInclusive = -1) { maxCount = maxCount == int.MaxValue ? maxCount - 1 : maxCount; @@ -98,7 +99,7 @@ protected override async Task ReadAllBackwardsInternal( long fromPositionExclusive, int maxCount, bool prefetch, - ReadNextAllPage readNext, + ReadNextAllPage readNext, CancellationToken cancellationToken) { maxCount = maxCount == int.MaxValue ? maxCount - 1 : maxCount; diff --git a/src/SqlStreamStore.MySql/MySqlStreamStore.Subscriptions.cs b/src/SqlStreamStore.MySql/MySqlStreamStore.Subscriptions.cs index 79e4078d2..3b9b92440 100644 --- a/src/SqlStreamStore.MySql/MySqlStreamStore.Subscriptions.cs +++ b/src/SqlStreamStore.MySql/MySqlStreamStore.Subscriptions.cs @@ -2,6 +2,7 @@ namespace SqlStreamStore { using System; using SqlStreamStore.Infrastructure; + using SqlStreamStore.Streams; using SqlStreamStore.Subscriptions; partial class MySqlStreamStore @@ -31,7 +32,7 @@ protected override IAllStreamSubscription SubscribeToAllInternal( HasCaughtUp hasCaughtUp, bool prefetchJsonData, string name) - => new AllStreamSubscription( + => new AllStreamSubscription( fromPosition, this, GetStoreObservable, diff --git a/src/SqlStreamStore.MySql/MySqlStreamStore.cs b/src/SqlStreamStore.MySql/MySqlStreamStore.cs index 40f4ef496..29d123c99 100644 --- a/src/SqlStreamStore.MySql/MySqlStreamStore.cs +++ b/src/SqlStreamStore.MySql/MySqlStreamStore.cs @@ -9,6 +9,7 @@ using SqlStreamStore.Infrastructure; using SqlStreamStore.Logging; using SqlStreamStore.MySqlScripts; + using SqlStreamStore.Streams; using SqlStreamStore.Subscriptions; using StreamStoreStore.Json; @@ -16,7 +17,7 @@ /// /// Represents a MySql stream store implementation. /// - public partial class MySqlStreamStore : StreamStoreBase + public partial class MySqlStreamStore : StreamStoreBase { private readonly MySqlStreamStoreSettings _settings; private readonly Func _createConnection; diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/ReadAll.sql b/src/SqlStreamStore.Postgres/PgSqlScripts/ReadAll.sql index 236c7e54b..cca4f083b 100644 --- a/src/SqlStreamStore.Postgres/PgSqlScripts/ReadAll.sql +++ b/src/SqlStreamStore.Postgres/PgSqlScripts/ReadAll.sql @@ -47,7 +47,6 @@ BEGIN WHEN _forwards AND _max_position > 0 THEN __schema__.messages.position >= _position AND __schema__.messages.position <= _max_position WHEN _forwards THEN __schema__.messages.position >= _position ELSE __schema__.messages.position <= _position END) - AND __schema__.messages.tx_id < txid_snapshot_xmin(txid_current_snapshot()) ORDER BY (CASE WHEN _forwards THEN __schema__.messages.position END), (CASE WHEN not _forwards THEN __schema__.messages.position END) DESC diff --git a/src/SqlStreamStore.Postgres/PostgresStreamStore.ReadAll.cs b/src/SqlStreamStore.Postgres/PostgresStreamStore.ReadAll.cs index f67b89215..a36c1db02 100644 --- a/src/SqlStreamStore.Postgres/PostgresStreamStore.ReadAll.cs +++ b/src/SqlStreamStore.Postgres/PostgresStreamStore.ReadAll.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.Data; using System.Data.Common; + using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -13,24 +14,55 @@ using SqlStreamStore.PgSqlScripts; using SqlStreamStore.Streams; + public class PostgresReadAllPage : ReadAllPage + { + public List TransactionIds { get; } + public PostgresReadAllPage( + long fromPosition, + long nextPosition, + bool isEnd, + ReadDirection direction, + ReadNextAllPage readNext, + StreamMessage[] messages = null, + string transactionSnapshot = null) : base(fromPosition, nextPosition, isEnd, direction, readNext, messages) + { + TransactionIds = ParseTransactionSnapshot(transactionSnapshot); + } + + private static List ParseTransactionSnapshot(string transactionSnapshot) + { + if (!string.IsNullOrWhiteSpace(transactionSnapshot)) + { + var splitResult = transactionSnapshot.Split(':'); + + if(splitResult.Length > 2 && !string.IsNullOrWhiteSpace(splitResult[2])) + { + return splitResult[2].Split(',').Select(x => Convert.ToInt64(x)).ToList(); + } + } + + return new List(); + } + } + partial class PostgresStreamStore { - protected override async Task ReadAllForwardsInternal( + protected override async Task ReadAllForwardsInternal( long fromPositionExclusive, int maxCount, bool prefetch, - ReadNextAllPage readNext, - CancellationToken cancellationToken, + ReadNextAllPage readNext, + CancellationToken cancellationToken, long fromMaxPositionInclusive = -1) { maxCount = maxCount == int.MaxValue ? maxCount - 1 : maxCount; var refcursorSql = new StringBuilder(); - using(var connection = await OpenConnection(cancellationToken)) - using(var transaction = connection.BeginTransaction()) + using (var connection = await OpenConnection(cancellationToken)) + using (var transaction = connection.BeginTransaction()) { - using(var command = BuildFunctionCommand( + using (var command = BuildFunctionCommand( _schema.ReadAll, transaction, Parameters.Count(maxCount + 1), @@ -38,25 +70,25 @@ protected override async Task ReadAllForwardsInternal( Parameters.ReadDirection(ReadDirection.Forward), Parameters.Prefetch(prefetch), Parameters.Position(fromMaxPositionInclusive))) - using(var reader = await command + using (var reader = await command .ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken) .ConfigureAwait(false)) { - while(await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) { refcursorSql.AppendLine(Schema.FetchAll(reader.GetString(0))); } } - using(var command2 = new NpgsqlCommand(refcursorSql.ToString(), transaction.Connection, transaction)) - using(var reader2 = await command2 + using (var command = new NpgsqlCommand(refcursorSql.ToString(), transaction.Connection, transaction)) + using (var reader = await command .ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken) .ConfigureAwait(false)) { - if(!reader2.HasRows) + if (!reader.HasRows) { - return new ReadAllPage( + return new PostgresReadAllPage( fromPositionExclusive, fromPositionExclusive, true, @@ -67,31 +99,31 @@ protected override async Task ReadAllForwardsInternal( var messages = new List<(StreamMessage message, int? maxAge)>(); - while(await reader2.ReadAsync(cancellationToken).ConfigureAwait(false)) + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) { - if(messages.Count == maxCount) + if (messages.Count == maxCount) { messages.Add(default); } else { - var streamIdInfo = new StreamIdInfo(reader2.GetString(0)); + var streamIdInfo = new StreamIdInfo(reader.GetString(0)); var (message, maxAge, _) = - await ReadAllStreamMessage(reader2, streamIdInfo.PostgresqlStreamId, prefetch); + await ReadAllStreamMessage(reader, streamIdInfo.PostgresqlStreamId, prefetch); messages.Add((message, maxAge)); } } - string txSnapshot = string.Empty; - await reader2.NextResultAsync(cancellationToken).ConfigureAwait(false); - while (await reader2.ReadAsync(cancellationToken).ConfigureAwait(false)) + string transactionSnapshot = string.Empty; + await reader.NextResultAsync(cancellationToken).ConfigureAwait(false); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) { - txSnapshot = await reader2.GetFieldValueAsync(0); + transactionSnapshot = await reader.GetFieldValueAsync(0, cancellationToken); } bool isEnd = true; - if(messages.Count == maxCount + 1) // An extra row was read, we're not at the end + if (messages.Count == maxCount + 1) // An extra row was read, we're not at the end { isEnd = false; messages.RemoveAt(maxCount); @@ -101,86 +133,105 @@ protected override async Task ReadAllForwardsInternal( var nextPosition = filteredMessages[filteredMessages.Count - 1].Position + 1; - return new ReadAllPage( + return new PostgresReadAllPage( fromPositionExclusive, nextPosition, isEnd, ReadDirection.Forward, readNext, filteredMessages.ToArray(), - txSnapshot); + transactionSnapshot); } } } - protected override async Task ReadAllBackwardsInternal( + protected override async Task ReadAllBackwardsInternal( long fromPositionExclusive, int maxCount, bool prefetch, - ReadNextAllPage readNext, + ReadNextAllPage readNext, CancellationToken cancellationToken) { maxCount = maxCount == int.MaxValue ? maxCount - 1 : maxCount; var ordinal = fromPositionExclusive == Position.End ? long.MaxValue : fromPositionExclusive; - using(var connection = await OpenConnection(cancellationToken)) - using(var transaction = connection.BeginTransaction()) - using(var command = BuildFunctionCommand( - _schema.ReadAll, - transaction, - Parameters.Count(maxCount + 1), - Parameters.Position(ordinal), - Parameters.ReadDirection(ReadDirection.Backward), - Parameters.Prefetch(prefetch))) - using(var reader = await command - .ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken) - .ConfigureAwait(false)) + var refcursorSql = new StringBuilder(); + + using (var connection = await OpenConnection(cancellationToken)) + using (var transaction = connection.BeginTransaction()) { - if(!reader.HasRows) + using (var command = BuildFunctionCommand( + _schema.ReadAll, + transaction, + Parameters.Count(maxCount + 1), + Parameters.Position(ordinal), + Parameters.ReadDirection(ReadDirection.Backward), + Parameters.Prefetch(prefetch), + Parameters.Position(-1))) { - // When reading backwards and there are no more items, then next position is LongPosition.Start, - // regardless of what the fromPosition is. - return new ReadAllPage( - Position.Start, - Position.Start, - true, - ReadDirection.Backward, - readNext, - Array.Empty()); - } + using (var reader = await command + .ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken) + .ConfigureAwait(false)) - var messages = new List<(StreamMessage message, int? maxAge)>(); + { + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + refcursorSql.AppendLine(Schema.FetchAll(reader.GetString(0))); + } + } + } - long lastOrdinal = 0; - while(await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + using (var command = new NpgsqlCommand(refcursorSql.ToString(), transaction.Connection, transaction)) + using (var reader = await command + .ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken) + .ConfigureAwait(false)) { - var streamIdInfo = new StreamIdInfo(reader.GetString(0)); - var (message, maxAge, position) = await ReadAllStreamMessage(reader, streamIdInfo.PostgresqlStreamId, prefetch); - messages.Add((message, maxAge)); + if (!reader.HasRows) + { + // When reading backwards and there are no more items, then next position is LongPosition.Start, + // regardless of what the fromPosition is. + return new PostgresReadAllPage( + Position.Start, + Position.Start, + true, + ReadDirection.Backward, + readNext, + Array.Empty()); + } - lastOrdinal = position; - } + var messages = new List<(StreamMessage message, int? maxAge)>(); - bool isEnd = true; - var nextPosition = lastOrdinal; + long lastOrdinal = 0; + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + var streamIdInfo = new StreamIdInfo(reader.GetString(0)); + var (message, maxAge, position) = await ReadAllStreamMessage(reader, streamIdInfo.PostgresqlStreamId, prefetch); + messages.Add((message, maxAge)); - if(messages.Count == maxCount + 1) // An extra row was read, we're not at the end - { - isEnd = false; - messages.RemoveAt(maxCount); - } + lastOrdinal = position; + } + + bool isEnd = true; + var nextPosition = lastOrdinal; - var filteredMessages = FilterExpired(messages); + if (messages.Count == maxCount + 1) // An extra row was read, we're not at the end + { + isEnd = false; + messages.RemoveAt(maxCount); + } + + var filteredMessages = FilterExpired(messages); - fromPositionExclusive = filteredMessages.Count > 0 ? filteredMessages[0].Position : 0; + fromPositionExclusive = filteredMessages.Count > 0 ? filteredMessages[0].Position : 0; - return new ReadAllPage( - fromPositionExclusive, - nextPosition, - isEnd, - ReadDirection.Backward, - readNext, - filteredMessages.ToArray()); + return new PostgresReadAllPage( + fromPositionExclusive, + nextPosition, + isEnd, + ReadDirection.Backward, + readNext, + filteredMessages.ToArray()); + } } } @@ -191,12 +242,12 @@ protected override async Task ReadAllBackwardsInternal( { async Task ReadString(int ordinal) { - if(reader.IsDBNull(ordinal)) + if (reader.IsDBNull(ordinal)) { return null; } - using(var textReader = reader.GetTextReader(ordinal)) + using (var textReader = reader.GetTextReader(ordinal)) { return await textReader.ReadToEndAsync().ConfigureAwait(false); } @@ -209,7 +260,7 @@ async Task ReadString(int ordinal) var type = reader.GetString(5); var jsonMetadata = await ReadString(6); - if(prefetch) + if (prefetch) { return ( new StreamMessage( diff --git a/src/SqlStreamStore.Postgres/PostgresStreamStore.Subscriptions.cs b/src/SqlStreamStore.Postgres/PostgresStreamStore.Subscriptions.cs index 24440fef5..481747968 100644 --- a/src/SqlStreamStore.Postgres/PostgresStreamStore.Subscriptions.cs +++ b/src/SqlStreamStore.Postgres/PostgresStreamStore.Subscriptions.cs @@ -2,6 +2,7 @@ { using System; using SqlStreamStore.Infrastructure; + using SqlStreamStore.Streams; using SqlStreamStore.Subscriptions; public partial class PostgresStreamStore @@ -32,7 +33,7 @@ protected override IAllStreamSubscription SubscribeToAllInternal( HasCaughtUp hasCaughtUp, bool prefetchJsonData, string name) - => new AllStreamSubscription( + => new AllStreamSubscription( fromPosition, this, GetStoreObservable, diff --git a/src/SqlStreamStore.Postgres/PostgresStreamStore.cs b/src/SqlStreamStore.Postgres/PostgresStreamStore.cs index 32da063ed..fc2ddb29d 100644 --- a/src/SqlStreamStore.Postgres/PostgresStreamStore.cs +++ b/src/SqlStreamStore.Postgres/PostgresStreamStore.cs @@ -14,7 +14,7 @@ /// /// Represents a PostgreSQL stream store implementation. /// - public partial class PostgresStreamStore : StreamStoreBase + public partial class PostgresStreamStore : StreamStoreBase { private readonly PostgresStreamStoreSettings _settings; private readonly Func _createConnection; @@ -259,5 +259,56 @@ public string GetSchemaCreationScript() { return _schema.Definition; } + + //protected override async Task HandleGap(T page, long fromPositionInclusive, int maxCount, bool prefetchJsonData, ReadNextAllPage readNext, CancellationToken cancellationToken) where T : PostgresReadAllPage + //{ + // if (page.Messages.Length == 0 || DateTime.UtcNow - page.Messages[page.Messages.Length - 1].CreatedUtc > TimeSpan.FromMinutes(5)) + // return page; + + // // TODO: FIXIT + // // Check for gap between last page and this. + // if (page.Messages[0].Position != fromPositionInclusive) + // { + // Logger.DebugFormat("Gap detected at lower page boundary. Potentially could have lost {lostMessageCount} events if the gap is transient", page.Messages[0].Position - fromPositionInclusive); + // page = await HandleGap(page, fromPositionInclusive, maxCount, prefetchJsonData, readNext, cancellationToken); + // //if (!page.IsEnd || page.Messages.Length == 1) + // // Logger.DebugFormat("Gap detected at lower page boundary. Potentially could have lost {lostMessageCount} events if the gap is transient", page.Messages[0].Position - fromPositionInclusive); + // //page = await ReloadAfterDelay(fromPositionInclusive, maxCount, prefetchJsonData, ReadNext, cancellationToken); + // } + + // // check for gap in messages collection + // for (int i = 0; i < page.Messages.Length - 1; i++) + // { + // var expectedNextPosition = page.Messages[i].Position + 1; + // if (expectedNextPosition != page.Messages[i + 1].Position) + // { + // Logger.InfoFormat("Gap detected in " + (page.IsEnd ? "last" : "(NOT the last)") + " page. Returning partial page {fromPosition}-{toPosition}", fromPositionInclusive, fromPositionInclusive + i + 1); + + // ReadAllPage requeryPage; + // var maxPosition = page.Messages[page.Messages.Length - 1].Position; + // do + // { + // requeryPage = await ReadAllForwardsInternal(fromPositionInclusive, maxCount, prefetchJsonData, readNext, cancellationToken, maxPosition); + // } while (page.TransactionIds.Intersect(requeryPage.TransactionIds).Any()); + + // return requeryPage; + + // // switched this to return the partial page, then re-issue load starting at gap + // // this speeds up the retry instead of taking a 3 second delay immediately + // //var messagesBeforeGap = new StreamMessage[i+1]; + // //page.Messages.Take(i+1).ToArray().CopyTo(messagesBeforeGap, 0); + // //return new ReadAllPage(page.FromPosition, maxPosition, page.IsEnd, page.Direction, ReadNext, messagesBeforeGap); + // } + // } + + // //ReadAllPage requeryPage; + // //var maxPosition = pageWithGap.Messages[pageWithGap.Messages.Length - 1].Position; + // //do + // //{ + // // requeryPage = await ReadAllForwardsInternal(fromPositionInclusive, maxCount, prefetchJsonData, readNext, cancellationToken, maxPosition); + // //} while (pageWithGap.TxSnapshot.CurrentTxIds.Intersect(requeryPage.TxSnapshot.CurrentTxIds).Any()); + + // return page; + //} } } diff --git a/src/SqlStreamStore.Sqlite/SqliteStreamStore.ReadAll.cs b/src/SqlStreamStore.Sqlite/SqliteStreamStore.ReadAll.cs index 97e4a893d..32d3f04f1 100644 --- a/src/SqlStreamStore.Sqlite/SqliteStreamStore.ReadAll.cs +++ b/src/SqlStreamStore.Sqlite/SqliteStreamStore.ReadAll.cs @@ -12,8 +12,9 @@ protected override async Task ReadAllForwardsInternal( long fromPositionExclusive, int maxCount, bool prefetch, - ReadNextAllPage readNext, - CancellationToken cancellationToken) + ReadNextAllPage readNext, + CancellationToken cancellationToken, + long fromMaxPositionInclusive = -1) { GuardAgainstDisposed(); cancellationToken.ThrowIfCancellationRequested(); @@ -83,7 +84,7 @@ protected override async Task ReadAllBackwardsInternal( long fromPosition, int maxCount, bool prefetch, - ReadNextAllPage readNext, + ReadNextAllPage readNext, CancellationToken cancellationToken) { GuardAgainstDisposed(); diff --git a/src/SqlStreamStore.Sqlite/SqliteStreamStore.Subscriptions.cs b/src/SqlStreamStore.Sqlite/SqliteStreamStore.Subscriptions.cs index cc670950d..c43bd05a6 100644 --- a/src/SqlStreamStore.Sqlite/SqliteStreamStore.Subscriptions.cs +++ b/src/SqlStreamStore.Sqlite/SqliteStreamStore.Subscriptions.cs @@ -2,6 +2,7 @@ namespace SqlStreamStore { using System; using SqlStreamStore.Infrastructure; + using SqlStreamStore.Streams; using SqlStreamStore.Subscriptions; public partial class SqliteStreamStore @@ -37,7 +38,7 @@ protected override IAllStreamSubscription SubscribeToAllInternal( bool prefetchJsonData, string name) { - return new AllStreamSubscription( + return new AllStreamSubscription( fromPosition, this, GetStoreObservable, diff --git a/src/SqlStreamStore.Sqlite/SqliteStreamStore.cs b/src/SqlStreamStore.Sqlite/SqliteStreamStore.cs index 7d53c5e2e..e843d1f7b 100644 --- a/src/SqlStreamStore.Sqlite/SqliteStreamStore.cs +++ b/src/SqlStreamStore.Sqlite/SqliteStreamStore.cs @@ -15,7 +15,7 @@ namespace SqlStreamStore using SqlStreamStore.Streams; using SqlStreamStore.Subscriptions; - public partial class SqliteStreamStore : StreamStoreBase + public partial class SqliteStreamStore : StreamStoreBase { private static readonly System.Collections.Concurrent.ConcurrentDictionary _scripts = new ConcurrentDictionary(); private static readonly Assembly s_assembly = typeof(SqliteStreamStore) diff --git a/src/SqlStreamStore/IReadonlyStreamStore.cs b/src/SqlStreamStore/IReadonlyStreamStore.cs index 3452e2e6d..853d25d73 100644 --- a/src/SqlStreamStore/IReadonlyStreamStore.cs +++ b/src/SqlStreamStore/IReadonlyStreamStore.cs @@ -6,65 +6,8 @@ using SqlStreamStore.Streams; using SqlStreamStore.Subscriptions; - /// - /// Represents a readonly stream store. - /// public interface IReadonlyStreamStore : IDisposable { - /// - /// Reads messages from all streams forwards. - /// - /// - /// The position to start reading from. Use to start from the beginning. - /// Note: messages that have expired will be filtered out. - /// - /// - /// The maximum number of messages to read (int.MaxValue is a bad idea). - /// - /// - /// Prefetches the message data as part of the page read. This means a single request to the server - /// but a higher payload size. - /// - /// - /// The cancellation instruction. - /// - /// - /// An presenting the result of the read. If all messages read have expired - /// then the message collection MAY be empty. - /// - Task ReadAllForwards( - long fromPositionInclusive, - int maxCount, - bool prefetchJsonData = true, - CancellationToken cancellationToken = default); - - /// - /// Reads messages from all streams backwards. - /// - /// - /// The position to start reading from. Use to start from the end. - /// Note: messages that have expired will be filtered out. - /// - /// - /// The maximum number of messages to read (int.MaxValue is a bad idea). - /// - /// - /// Prefetches the message data as part of the page read. This means a single request to the server - /// but a higher payload size. - /// - /// - /// The cancellation instruction. - /// - /// - /// An presenting the result of the read. If all messages read have expired - /// then the message collection MAY be empty. - /// - Task ReadAllBackwards( - long fromPositionInclusive, - int maxCount, - bool prefetchJsonData = true, - CancellationToken cancellationToken = default); - /// /// Reads messages from a stream forwards. /// @@ -304,4 +247,64 @@ Task ListStreams( event Action OnDispose; } + + /// + /// Represents a readonly stream store. + /// + public interface IReadonlyStreamStore : IReadonlyStreamStore where TReadAllPage : IReadAllPage + { + /// + /// Reads messages from all streams forwards. + /// + /// + /// The position to start reading from. Use to start from the beginning. + /// Note: messages that have expired will be filtered out. + /// + /// + /// The maximum number of messages to read (int.MaxValue is a bad idea). + /// + /// + /// Prefetches the message data as part of the page read. This means a single request to the server + /// but a higher payload size. + /// + /// + /// The cancellation instruction. + /// + /// + /// An presenting the result of the read. If all messages read have expired + /// then the message collection MAY be empty. + /// + Task ReadAllForwards( + long fromPositionInclusive, + int maxCount, + bool prefetchJsonData = true, + CancellationToken cancellationToken = default); + + /// + /// Reads messages from all streams backwards. + /// + /// + /// The position to start reading from. Use to start from the end. + /// Note: messages that have expired will be filtered out. + /// + /// + /// The maximum number of messages to read (int.MaxValue is a bad idea). + /// + /// + /// Prefetches the message data as part of the page read. This means a single request to the server + /// but a higher payload size. + /// + /// + /// The cancellation instruction. + /// + /// + /// An presenting the result of the read. If all messages read have expired + /// then the message collection MAY be empty. + /// + Task ReadAllBackwards( + long fromPositionInclusive, + int maxCount, + bool prefetchJsonData = true, + CancellationToken cancellationToken = default); + } } \ No newline at end of file diff --git a/src/SqlStreamStore/IStreamStore.cs b/src/SqlStreamStore/IStreamStore.cs index aa21cc89c..a4fb388f5 100644 --- a/src/SqlStreamStore/IStreamStore.cs +++ b/src/SqlStreamStore/IStreamStore.cs @@ -121,4 +121,9 @@ Task SetStreamMetadata( string metadataJson = null, CancellationToken cancellationToken = default); } + + // TODO + public interface IStreamStore : IStreamStore, IReadonlyStreamStore where TReadPage : IReadAllPage + { + } } diff --git a/src/SqlStreamStore/Infrastructure/ReadonlyStreamStoreBase.cs b/src/SqlStreamStore/Infrastructure/ReadonlyStreamStoreBase.cs index 2e7cf92d8..d58777fcc 100644 --- a/src/SqlStreamStore/Infrastructure/ReadonlyStreamStoreBase.cs +++ b/src/SqlStreamStore/Infrastructure/ReadonlyStreamStoreBase.cs @@ -1,5 +1,3 @@ -using System.Linq; - namespace SqlStreamStore.Infrastructure { using System; @@ -15,7 +13,7 @@ namespace SqlStreamStore.Infrastructure /// /// Represents a base implementation of a readonly stream store. /// - public abstract class ReadonlyStreamStoreBase : IReadonlyStreamStore + public abstract class ReadonlyStreamStoreBase : IReadonlyStreamStore where TReadAllPage : IReadAllPage { private const int DefaultReloadInterval = 3000; protected readonly GetUtcNow GetUtcNow; @@ -44,7 +42,7 @@ protected ReadonlyStreamStoreBase(GetUtcNow getUtcNow, string logName) _disableMetadataCache = true; } - public async Task ReadAllForwards( + public async Task ReadAllForwards( long fromPositionInclusive, int maxCount, bool prefetchJsonData, @@ -59,12 +57,14 @@ public async Task ReadAllForwards( Logger.DebugFormat("ReadAllForwards from position {fromPositionInclusive} with max count " + "{maxCount}.", fromPositionInclusive, maxCount); - Task ReadNext(long nextPosition, CancellationToken ct) => ReadAllForwards(nextPosition, maxCount, prefetchJsonData, ct); + Task ReadNext(long nextPosition, CancellationToken ct) => ReadAllForwards(nextPosition, maxCount, prefetchJsonData, ct); var page = await ReadAllForwardsInternal(fromPositionInclusive, maxCount, prefetchJsonData, ReadNext, cancellationToken) .ConfigureAwait(false); + var pageHandled = await HandleGap(page, fromPositionInclusive, maxCount, prefetchJsonData, ReadNext, cancellationToken); + return await FilterExpired(pageHandled, ReadNext, cancellationToken).ConfigureAwait(false); //// https://github.com/damianh/SqlStreamStore/issues/31 //// Under heavy parallel load, gaps may appear in the position sequence due to sequence @@ -78,48 +78,132 @@ public async Task ReadAllForwards( // only short circuit now for empty pages or 'old' pages, where // 'old' is defined as pages whose last message is older than _positionWriteDelayThreshold ago - if(page.Messages.Length == 0 || ((DateTime.UtcNow - page.Messages[page.Messages.Length-1].CreatedUtc) > TimeSpan.FromMinutes(5))) - return await FilterExpired(page, ReadNext, cancellationToken).ConfigureAwait(false); - + //if (page.Messages.Length == 0 || ((DateTime.UtcNow - page.Messages[page.Messages.Length - 1].CreatedUtc) > TimeSpan.FromMinutes(5))) + // return await FilterExpired(page, ReadNext, cancellationToken).ConfigureAwait(false); + //// TODO: FIXIT //// Check for gap between last page and this. - //if(page.Messages[0].Position != fromPositionInclusive) + //if (page.Messages[0].Position != fromPositionInclusive) //{ - // if(!page.IsEnd || page.Messages.Length == 1) - // Logger.DebugFormat("Gap detected at lower page boundary. Potentially could have lost {lostMessageCount} events if the gap is transient", page.Messages[0].Position - fromPositionInclusive); - // page = await ReloadAfterDelay(fromPositionInclusive, maxCount, prefetchJsonData, ReadNext, cancellationToken); + // Logger.DebugFormat("Gap detected at lower page boundary. Potentially could have lost {lostMessageCount} events if the gap is transient", page.Messages[0].Position - fromPositionInclusive); + // page = await HandleGap(page, fromPositionInclusive, maxCount, prefetchJsonData, ReadNext, cancellationToken); + // //if (!page.IsEnd || page.Messages.Length == 1) + // // Logger.DebugFormat("Gap detected at lower page boundary. Potentially could have lost {lostMessageCount} events if the gap is transient", page.Messages[0].Position - fromPositionInclusive); + // //page = await ReloadAfterDelay(fromPositionInclusive, maxCount, prefetchJsonData, ReadNext, cancellationToken); //} + //// check for gap in messages collection + //for (int i = 0; i < page.Messages.Length - 1; i++) + //{ + // var expectedNextPosition = page.Messages[i].Position + 1; + // if (expectedNextPosition != page.Messages[i + 1].Position) + // { + // Logger.InfoFormat("Gap detected in " + (page.IsEnd ? "last" : "(NOT the last)") + " page. Returning partial page {fromPosition}-{toPosition}", fromPositionInclusive, fromPositionInclusive + i + 1); + + // ReadAllPage requeryPage; + // var maxPosition = page.Messages[page.Messages.Length - 1].Position; + // do + // { + // requeryPage = await ReadAllForwardsInternal(fromPositionInclusive, maxCount, prefetchJsonData, ReadNext, cancellationToken, maxPosition); + // } while (page.TxSnapshot.CurrentTxIds.Intersect(requeryPage.TxSnapshot.CurrentTxIds).Any()); + + // return await FilterExpired(requeryPage, ReadNext, cancellationToken).ConfigureAwait(false); + + // // switched this to return the partial page, then re-issue load starting at gap + // // this speeds up the retry instead of taking a 3 second delay immediately + // //var messagesBeforeGap = new StreamMessage[i+1]; + // //page.Messages.Take(i+1).ToArray().CopyTo(messagesBeforeGap, 0); + // //return new ReadAllPage(page.FromPosition, maxPosition, page.IsEnd, page.Direction, ReadNext, messagesBeforeGap); + // } + //} + + //return await FilterExpired(page, ReadNext, cancellationToken).ConfigureAwait(false); + } + + //protected virtual async Task HandleGap(T page, long fromPositionInclusive, int maxCount, bool prefetchJsonData, ReadNextAllPage readNext, CancellationToken cancellationToken) where T : ReadAllPage + //{ + // if (page.Messages.Length == 0 || DateTime.UtcNow - page.Messages[page.Messages.Length - 1].CreatedUtc > TimeSpan.FromMinutes(5)) + // return page; + + + // // TODO: FIXIT + // // Check for gap between last page and this. + // if (page.Messages[0].Position != fromPositionInclusive) + // { + // Logger.DebugFormat("Gap detected at lower page boundary. Potentially could have lost {lostMessageCount} events if the gap is transient", page.Messages[0].Position - fromPositionInclusive); + // page = await HandleGap(page, fromPositionInclusive, maxCount, prefetchJsonData, readNext, cancellationToken); + // //if (!page.IsEnd || page.Messages.Length == 1) + // // Logger.DebugFormat("Gap detected at lower page boundary. Potentially could have lost {lostMessageCount} events if the gap is transient", page.Messages[0].Position - fromPositionInclusive); + // //page = await ReloadAfterDelay(fromPositionInclusive, maxCount, prefetchJsonData, ReadNext, cancellationToken); + // } + + // // check for gap in messages collection + // for (int i = 0; i < page.Messages.Length - 1; i++) + // { + // var expectedNextPosition = page.Messages[i].Position + 1; + // if (expectedNextPosition != page.Messages[i + 1].Position) + // { + // Logger.InfoFormat("Gap detected in " + (page.IsEnd ? "last" : "(NOT the last)") + " page. Returning partial page {fromPosition}-{toPosition}", fromPositionInclusive, fromPositionInclusive + i + 1); + + // ReadAllPage requeryPage; + // var maxPosition = page.Messages[page.Messages.Length - 1].Position; + // do + // { + // requeryPage = await ReadAllForwardsInternal(fromPositionInclusive, maxCount, prefetchJsonData, readNext, cancellationToken, maxPosition); + // } while (page.TxSnapshot.CurrentTxIds.Intersect(requeryPage.TxSnapshot.CurrentTxIds).Any()); + + // return requeryPage; + + // // switched this to return the partial page, then re-issue load starting at gap + // // this speeds up the retry instead of taking a 3 second delay immediately + // //var messagesBeforeGap = new StreamMessage[i+1]; + // //page.Messages.Take(i+1).ToArray().CopyTo(messagesBeforeGap, 0); + // //return new ReadAllPage(page.FromPosition, maxPosition, page.IsEnd, page.Direction, ReadNext, messagesBeforeGap); + // } + // } + + // //ReadAllPage requeryPage; + // //var maxPosition = pageWithGap.Messages[pageWithGap.Messages.Length - 1].Position; + // //do + // //{ + // // requeryPage = await ReadAllForwardsInternal(fromPositionInclusive, maxCount, prefetchJsonData, readNext, cancellationToken, maxPosition); + // //} while (pageWithGap.TxSnapshot.CurrentTxIds.Intersect(requeryPage.TxSnapshot.CurrentTxIds).Any()); + + // return page; + //} + + protected virtual async Task HandleGap(TReadAllPage page, long fromPositionInclusive, int maxCount, bool prefetchJsonData, ReadNextAllPage readNext, CancellationToken cancellationToken) + { + // https://github.com/damianh/SqlStreamStore/issues/31 + // Under heavy parallel load, gaps may appear in the position sequence due to sequence + // number reservation of in-flight transactions. + // Here we check if there are any gaps, and in the unlikely event there is, we delay a little bit + // and re-issue the read. This is expected + if(!page.IsEnd || page.Messages.Length <= 1) + { + return page; + } + + // Check for gap between last page and this. + if (page.Messages[0].Position != fromPositionInclusive) + { + return await ReloadAfterDelay(fromPositionInclusive, maxCount, prefetchJsonData, readNext, cancellationToken); + } + // check for gap in messages collection for(int i = 0; i < page.Messages.Length - 1; i++) { - var expectedNextPosition = page.Messages[i].Position + 1; - if(expectedNextPosition != page.Messages[i + 1].Position) + if(page.Messages[i].Position + 1 != page.Messages[i + 1].Position) { - Logger.InfoFormat("Gap detected in "+(page.IsEnd?"last":"(NOT the last)")+" page. Returning partial page {fromPosition}-{toPosition}", fromPositionInclusive, fromPositionInclusive+i+1); - - ReadAllPage requeryPage; - var maxPosition = page.Messages[page.Messages.Length - 1].Position; - do - { - requeryPage = await ReadAllForwardsInternal(fromPositionInclusive, maxCount, prefetchJsonData, ReadNext, cancellationToken, maxPosition); - } while(page.TxSnapshot.CurrentTxIds.Intersect(requeryPage.TxSnapshot.CurrentTxIds).Any()); - - return await FilterExpired(requeryPage, ReadNext, cancellationToken).ConfigureAwait(false); - - // switched this to return the partial page, then re-issue load starting at gap - // this speeds up the retry instead of taking a 3 second delay immediately - //var messagesBeforeGap = new StreamMessage[i+1]; - //page.Messages.Take(i+1).ToArray().CopyTo(messagesBeforeGap, 0); - //return new ReadAllPage(page.FromPosition, maxPosition, page.IsEnd, page.Direction, ReadNext, messagesBeforeGap); + return await ReloadAfterDelay(fromPositionInclusive, maxCount, prefetchJsonData, readNext, cancellationToken); } } - return await FilterExpired(page, ReadNext, cancellationToken).ConfigureAwait(false); + return page; } - public async Task ReadAllBackwards( + public async Task ReadAllBackwards( long fromPositionInclusive, int maxCount, bool prefetchJsonData, @@ -136,7 +220,7 @@ public async Task ReadAllBackwards( fromPositionInclusive, maxCount); - ReadNextAllPage readNext = (nextPosition, ct) => ReadAllBackwards(nextPosition, maxCount, prefetchJsonData, ct); + ReadNextAllPage readNext = (nextPosition, ct) => ReadAllBackwards(nextPosition, maxCount, prefetchJsonData, ct); var page = await ReadAllBackwardsInternal(fromPositionInclusive, maxCount, prefetchJsonData, readNext, cancellationToken); return await FilterExpired(page, readNext, cancellationToken); } @@ -314,19 +398,19 @@ public void Dispose() public event Action OnDispose; - protected abstract Task ReadAllForwardsInternal( + protected abstract Task ReadAllForwardsInternal( long fromPositionExclusive, int maxCount, bool prefetch, - ReadNextAllPage readNext, + ReadNextAllPage readNext, CancellationToken cancellationToken, long fromMaxPositionInclusive = -1); - protected abstract Task ReadAllBackwardsInternal( + protected abstract Task ReadAllBackwardsInternal( long fromPositionExclusive, int maxCount, bool prefetch, - ReadNextAllPage readNext, + ReadNextAllPage readNext, CancellationToken cancellationToken); protected abstract Task ReadStreamForwardsInternal( @@ -377,11 +461,11 @@ protected abstract Task ListStreamsInternal( CancellationToken cancellationToken); protected virtual void Dispose(bool disposing) - {} + { } protected void GuardAgainstDisposed() { - if(_isDisposed) + if (_isDisposed) { throw new ObjectDisposedException(GetType().Name); } @@ -389,11 +473,11 @@ protected void GuardAgainstDisposed() protected abstract void PurgeExpiredMessage(StreamMessage streamMessage); - private async Task ReloadAfterDelay( + private async Task ReloadAfterDelay( long fromPositionInclusive, int maxCount, bool prefetch, - ReadNextAllPage readNext, + ReadNextAllPage readNext, CancellationToken cancellationToken) { Logger.Info("ReadAllForwards: gap detected in position, reloading after {DefaultReloadInterval}ms, position: {fromPositionInclusive}", DefaultReloadInterval, fromPositionInclusive); @@ -408,7 +492,7 @@ private async Task FilterExpired( ReadNextStreamPage readNext, CancellationToken cancellationToken) { - if(page.StreamId.StartsWith("$")) + if (page.StreamId.StartsWith("$")) { return page; } @@ -422,9 +506,9 @@ private async Task FilterExpired( } var currentUtc = GetUtcNow(); var valid = new List(); - foreach(var message in page.Messages) + foreach (var message in page.Messages) { - if(message.CreatedUtc.AddSeconds(maxAge.Value) > currentUtc) + if (message.CreatedUtc.AddSeconds(maxAge.Value) > currentUtc) { valid.Add(message); } @@ -446,12 +530,12 @@ private async Task FilterExpired( valid.ToArray()); } - private async Task FilterExpired( - ReadAllPage readAllPage, - ReadNextAllPage readNext, + private async Task FilterExpired(TReadAllPage readAllPage, + ReadNextAllPage readNext, CancellationToken cancellationToken) { - if(_disableMetadataCache) + return readAllPage; + if (_disableMetadataCache) { return readAllPage; } @@ -459,7 +543,7 @@ private async Task FilterExpired( var currentUtc = GetUtcNow(); foreach (var streamMessage in readAllPage.Messages) { - if(streamMessage.StreamId.StartsWith("$")) + if (streamMessage.StreamId.StartsWith("$")) { valid.Add(streamMessage); continue; @@ -481,13 +565,13 @@ private async Task FilterExpired( PurgeExpiredMessage(streamMessage); } } - return new ReadAllPage( - readAllPage.FromPosition, - readAllPage.NextPosition, - readAllPage.IsEnd, - readAllPage.Direction, - readNext, - valid.ToArray()); + //return new ReadAllPage( + // readAllPage.FromPosition, + // readAllPage.NextPosition, + // readAllPage.IsEnd, + // readAllPage.Direction, + // readNext, + // valid.ToArray()); } protected List FilterExpired(List<(StreamMessage StreamMessage, int? MaxAge)> messages) diff --git a/src/SqlStreamStore/Infrastructure/StreamStoreBase.cs b/src/SqlStreamStore/Infrastructure/StreamStoreBase.cs index b4b35c189..870a246a4 100644 --- a/src/SqlStreamStore/Infrastructure/StreamStoreBase.cs +++ b/src/SqlStreamStore/Infrastructure/StreamStoreBase.cs @@ -12,7 +12,7 @@ namespace SqlStreamStore.Infrastructure /// Represents a base implementation of a stream store. It's primary purpose is to handle /// common needs across all derived implementations such as guard clauses and logging. /// - public abstract class StreamStoreBase : ReadonlyStreamStoreBase, IStreamStore + public abstract class StreamStoreBase : ReadonlyStreamStoreBase, IStreamStore where TReadAllPage : IReadAllPage { private readonly TaskQueue _taskQueue = new TaskQueue(); diff --git a/src/SqlStreamStore/StreamStoreExtensions.cs b/src/SqlStreamStore/StreamStoreExtensions.cs index d8840dcb2..263cac63d 100644 --- a/src/SqlStreamStore/StreamStoreExtensions.cs +++ b/src/SqlStreamStore/StreamStoreExtensions.cs @@ -29,11 +29,11 @@ public static class StreamStoreExtensions /// An presenting the result of the read. If all messages read have expired /// then the message collection MAY be empty. /// - public static Task ReadAllForwards( - this IReadonlyStreamStore readonlyStreamStore, + public static Task ReadAllForwards( + this IReadonlyStreamStore readonlyStreamStore, long fromPositionInclusive, int maxCount, - CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default) where TReadAllPage : IReadAllPage { return readonlyStreamStore.ReadAllForwards( fromPositionInclusive, @@ -61,11 +61,11 @@ public static Task ReadAllForwards( /// An presenting the result of the read. If all messages read have expired /// then the message collection MAY be empty. /// - public static Task ReadAllBackwards( - this IReadonlyStreamStore readonlyStreamStore, + public static Task ReadAllBackwards( + this IReadonlyStreamStore readonlyStreamStore, long fromPositionInclusive, int maxCount, - CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default) where TReadAllPage : ReadAllPage, new() { return readonlyStreamStore.ReadAllBackwards( fromPositionInclusive, @@ -96,12 +96,12 @@ public static Task ReadAllBackwards( /// An represent the result of the operation. If all the messages read /// have expired then the message collection MAY be empty. /// - public static Task ReadStreamForwards( - this IReadonlyStreamStore readonlyStreamStore, + public static Task ReadStreamForwards( + this IReadonlyStreamStore readonlyStreamStore, string streamId, int fromVersionInclusive, int maxCount, - CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default) where TReadAllPage : ReadAllPage, new() { return readonlyStreamStore.ReadStreamForwards( streamId, diff --git a/src/SqlStreamStore/Streams/ReadAllPage.cs b/src/SqlStreamStore/Streams/ReadAllPage.cs index ae9ffb62f..2e1d51c30 100644 --- a/src/SqlStreamStore/Streams/ReadAllPage.cs +++ b/src/SqlStreamStore/Streams/ReadAllPage.cs @@ -1,44 +1,55 @@ namespace SqlStreamStore.Streams { - using System; - using System.Collections.Generic; - using System.Linq; using System.Threading; using System.Threading.Tasks; + + public interface IReadAllPage + { + long FromPosition { get; } + long NextPosition { get; } + bool IsEnd { get; } + ReadDirection Direction { get; } + StreamMessage[] Messages { get; } + } + + public class ReadAllPage : ReadAllPage + { + public ReadAllPage(long fromPosition, long nextPosition, bool isEnd, ReadDirection direction, ReadNextAllPage readNext, StreamMessage[] messages = null) : base(fromPosition, nextPosition, isEnd, direction, readNext, messages) + { } + } + /// /// Represents the result of a read of all streams. /// - public sealed class ReadAllPage + public abstract class ReadAllPage : IReadAllPage where TReadAllPage : IReadAllPage { - private readonly ReadNextAllPage _readNext; + private readonly ReadNextAllPage _readNext; /// /// A long representing the position where this page was read from. /// - public readonly long FromPosition; + public long FromPosition { get; } /// /// A long representing the position where the next page should be read from. /// - public readonly long NextPosition; + public long NextPosition { get; } /// /// True if page reach end of the all stream at time of reading. Otherwise false. /// - public readonly bool IsEnd; + public bool IsEnd { get; } /// /// The direction of the the read request. /// - public readonly ReadDirection Direction; + public ReadDirection Direction { get; } /// /// The collection of s returned as part of the read. /// - public readonly StreamMessage[] Messages; - - public readonly TxSnapshot TxSnapshot; + public StreamMessage[] Messages { get; } /// /// Initializes a new instance of @@ -49,14 +60,13 @@ public sealed class ReadAllPage /// The direction of the the read request. /// An operation to read the next page of messages. /// The collection messages read. - public ReadAllPage( + protected ReadAllPage( long fromPosition, long nextPosition, bool isEnd, ReadDirection direction, - ReadNextAllPage readNext, - StreamMessage[] messages = null, - string txSnapshot = null) + ReadNextAllPage readNext, + StreamMessage[] messages = null) { FromPosition = fromPosition; NextPosition = nextPosition; @@ -64,7 +74,6 @@ public ReadAllPage( Direction = direction; _readNext = readNext; Messages = messages ?? new StreamMessage[0]; - TxSnapshot = txSnapshot == null ? null : new TxSnapshot(txSnapshot); } /// @@ -79,28 +88,9 @@ public override string ToString() /// /// A token to cancel the operations. /// A task the represents the asyncronous operation. - public Task ReadNext(CancellationToken cancellationToken = default) + public Task ReadNext(CancellationToken cancellationToken = default) { return _readNext(NextPosition, cancellationToken); } } - - public class TxSnapshot - { - private readonly long MinTx; - private readonly long MaxTx; - - public readonly List CurrentTxIds = new List(); - public TxSnapshot(string txSnapshot) - { - var splitted = txSnapshot.Split(':'); - MinTx = Convert.ToInt64(splitted[0]); - MaxTx = Convert.ToInt64(splitted[1]); - - if(splitted.Length > 2 && !string.IsNullOrWhiteSpace(splitted[2])) - { - CurrentTxIds = splitted[2].Split(',').Select(x => Convert.ToInt64(x)).ToList(); - } - } - } } \ No newline at end of file diff --git a/src/SqlStreamStore/Streams/ReadNextAllPage.cs b/src/SqlStreamStore/Streams/ReadNextAllPage.cs index b80ba8522..77d945c16 100644 --- a/src/SqlStreamStore/Streams/ReadNextAllPage.cs +++ b/src/SqlStreamStore/Streams/ReadNextAllPage.cs @@ -9,5 +9,6 @@ /// The position to read from. /// A token to cancel the operation. /// A task that represents the result of the operation.z - public delegate Task ReadNextAllPage(long nextPosition, CancellationToken cancellationToken); + //public delegate Task ReadNextAllPage(long nextPosition, CancellationToken cancellationToken) where TReadAllPage : ReadAllPage; + public delegate Task ReadNextAllPage(long nextPosition, CancellationToken cancellationToken) where TReadAllPage : IReadAllPage; } \ No newline at end of file diff --git a/src/SqlStreamStore/Subscriptions/AllStreamSubscription.cs b/src/SqlStreamStore/Subscriptions/AllStreamSubscription.cs index 32ae0f214..c16c1de1e 100644 --- a/src/SqlStreamStore/Subscriptions/AllStreamSubscription.cs +++ b/src/SqlStreamStore/Subscriptions/AllStreamSubscription.cs @@ -13,13 +13,13 @@ /// Represents a subscription to all streams. /// - public sealed class AllStreamSubscription : IAllStreamSubscription + public sealed class AllStreamSubscription : IAllStreamSubscription where TReadAllPage : IReadAllPage { public const int DefaultPageSize = 10; private static readonly ILog s_logger = LogProvider.GetLogger("SqlStreamStore.Subscriptions.AllStreamSubscription"); private int _pageSize = DefaultPageSize; private long _nextPosition; - private readonly IReadonlyStreamStore _readonlyStreamStore; + private readonly IReadonlyStreamStore _readonlyStreamStore; private readonly AllStreamMessageReceived _streamMessageReceived; private readonly bool _prefetchJsonData; private readonly HasCaughtUp _hasCaughtUp; @@ -32,7 +32,7 @@ public sealed class AllStreamSubscription : IAllStreamSubscription public AllStreamSubscription( long? continueAfterPosition, - IReadonlyStreamStore readonlyStreamStore, + IReadonlyStreamStore readonlyStreamStore, IObservable streamStoreAppendedNotification, AllStreamMessageReceived streamMessageReceived, AllSubscriptionDropped subscriptionDropped, @@ -176,9 +176,9 @@ private async Task Initialize() _nextPosition = headPosition == 0 ? 0 : headPosition + 1; } - private async Task Pull() + private async Task Pull() { - ReadAllPage readAllPage; + TReadAllPage readAllPage; try { readAllPage = await _readonlyStreamStore @@ -204,7 +204,7 @@ private async Task Pull() return readAllPage; } - private async Task Push(ReadAllPage page) + private async Task Push(TReadAllPage page) { foreach (var message in page.Messages) { diff --git a/src/SqlStreamStore/Subscriptions/CreateStreamStoreNotifier.cs b/src/SqlStreamStore/Subscriptions/CreateStreamStoreNotifier.cs index 9cca6b22a..5be05e9c3 100644 --- a/src/SqlStreamStore/Subscriptions/CreateStreamStoreNotifier.cs +++ b/src/SqlStreamStore/Subscriptions/CreateStreamStoreNotifier.cs @@ -1,6 +1,6 @@ namespace SqlStreamStore.Subscriptions { - using SqlStreamStore; + using SqlStreamStore.Streams; /// /// Represents an operation to create a stream store notifier. From c3f9eea442f2d4bd8bae730c38746967a4f62786 Mon Sep 17 00:00:00 2001 From: Arne Schoonvliet Date: Wed, 2 Feb 2022 15:19:59 +0100 Subject: [PATCH 04/13] WIP fixing it --- build/build.csproj | 2 +- src/Directory.Build.props | 2 +- src/LoadTests/AppendsReadsDeadlocks.cs | 2 +- src/LoadTests/LoadTest.cs | 6 +- src/LoadTests/LoadTests.csproj | 2 +- src/LoadTests/ReadAllBackwards.cs | 2 +- src/LoadTests/ReadAllForwards.cs | 2 +- src/LoadTests/TestGaps.cs | 6 +- .../SqlStreamStore.HAL.DevServer.csproj | 2 +- .../SqlStreamStoreFactory.cs | 14 +- .../SqlStreamStore.HAL.csproj | 2 +- .../SqlStreamStoreHalMiddleware.cs | 2 +- .../SqlStreamStore.Http.csproj | 2 +- .../SqlStreamStore.MsSql.csproj | 2 +- .../SqlStreamStore.MySql.csproj | 2 +- .../PostgresStreamStore.ReadAll.cs | 9 +- .../PostgresStreamStore.cs | 57 ++ .../SqlStreamStore.Postgres.csproj | 2 +- ...treamStore.SchemaCreationScriptTool.csproj | 2 +- .../SqlStreamStore.Sqlite.csproj | 2 +- .../SqliteStreamStore.ReadAll.cs | 22 +- .../SqlStreamStore.TestUtils.csproj | 2 +- src/SqlStreamStore/IReadonlyStreamStore.cs | 7 +- src/SqlStreamStore/IStreamStore.cs | 2 +- .../InMemory/InMemoryAllStream.cs | 7 + src/SqlStreamStore/InMemory/InMemoryStream.cs | 225 +++++ .../InMemory/InMemoryStreamMessage.cs | 36 + .../InMemory/InMemoryStreamStore.cs | 778 ++++++++++++++++++ .../Infrastructure/ReadonlyStreamStoreBase.cs | 33 +- .../Infrastructure/StreamStoreBase.cs | 2 +- src/SqlStreamStore/SqlStreamStore.csproj | 3 +- src/SqlStreamStore/StreamStoreExtensions.cs | 7 +- src/SqlStreamStore/Streams/ReadAllPage.cs | 29 +- src/SqlStreamStore/Streams/ReadNextAllPage.cs | 3 + tests/Directory.Build.props | 2 +- .../AcceptanceTests.ReadAll.cs | 2 +- .../AcceptanceTests.cs | 2 +- .../IStreamStoreFixture.cs | 2 +- .../SqlStreamStore.Http.Tests.csproj | 2 +- .../SqlStreamStore.MsSql.Tests.csproj | 2 +- .../SqlStreamStore.MsSql.V3.Tests.csproj | 2 +- .../SqlStreamStore.MySql.Tests.csproj | 2 +- .../PostgresStreamStoreFixture.cs | 2 +- .../SqlStreamStore.Postgres.Tests.csproj | 2 +- .../SqlStreamStore.Sqlite.Tests.csproj | 2 +- 45 files changed, 1193 insertions(+), 107 deletions(-) create mode 100644 src/SqlStreamStore/InMemory/InMemoryAllStream.cs create mode 100644 src/SqlStreamStore/InMemory/InMemoryStream.cs create mode 100644 src/SqlStreamStore/InMemory/InMemoryStreamMessage.cs create mode 100644 src/SqlStreamStore/InMemory/InMemoryStreamStore.cs diff --git a/build/build.csproj b/build/build.csproj index 911b97451..824a9caaa 100644 --- a/build/build.csproj +++ b/build/build.csproj @@ -2,7 +2,7 @@ Exe - netcoreapp3.1 + net5.0 diff --git a/src/Directory.Build.props b/src/Directory.Build.props index c3a6e9c51..df32468c1 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -7,7 +7,7 @@ true cqrs;event-sourcing;event-store;stream-store $(NoWarn);1701;1702;1705;1591 - 7.3 + 9.0 v 1.2 full diff --git a/src/LoadTests/AppendsReadsDeadlocks.cs b/src/LoadTests/AppendsReadsDeadlocks.cs index e129202ad..8f6eeb09f 100644 --- a/src/LoadTests/AppendsReadsDeadlocks.cs +++ b/src/LoadTests/AppendsReadsDeadlocks.cs @@ -55,7 +55,7 @@ await dictStreamMessage.ForEachAsync(chunkSize, } } - public async Task GetManyPagesAsync(IStreamStore store, int chunksCount, int batchSize) + public async Task GetManyPagesAsync(IStreamStore store, int chunksCount, int batchSize) { long start = 0; //var events = new List(); diff --git a/src/LoadTests/LoadTest.cs b/src/LoadTests/LoadTest.cs index 80c9954c6..17bbac63f 100644 --- a/src/LoadTests/LoadTest.cs +++ b/src/LoadTests/LoadTest.cs @@ -5,14 +5,16 @@ using System.Threading.Tasks; using EasyConsole; using SqlStreamStore; + using SqlStreamStore.Streams; + public abstract class LoadTest { public abstract Task Run(CancellationToken cancellationToken); - protected async Task<(IStreamStore, Action)> GetStore(CancellationToken cancellationToken) + protected async Task<(IStreamStore, Action)> GetStore(CancellationToken cancellationToken) { - IStreamStore streamStore = null; + IStreamStore streamStore = null; IDisposable disposable = null; Output.WriteLine(ConsoleColor.Yellow, "Store type:"); diff --git a/src/LoadTests/LoadTests.csproj b/src/LoadTests/LoadTests.csproj index c4095b798..d84344e19 100644 --- a/src/LoadTests/LoadTests.csproj +++ b/src/LoadTests/LoadTests.csproj @@ -1,6 +1,6 @@  - netcoreapp3.1 + net5.0 win10-x64;osx.10.11-x64;ubuntu.16.10-x64;ubuntu.14.04-x64 exe false diff --git a/src/LoadTests/ReadAllBackwards.cs b/src/LoadTests/ReadAllBackwards.cs index d6ca7144a..d064281ac 100644 --- a/src/LoadTests/ReadAllBackwards.cs +++ b/src/LoadTests/ReadAllBackwards.cs @@ -28,7 +28,7 @@ public override async Task Run(CancellationToken ct) var stopwatch = Stopwatch.StartNew(); int count = 0; var position = Position.End; - ReadAllPage page; + IReadAllPage page; do { page = await streamStore.ReadAllBackwards(position, diff --git a/src/LoadTests/ReadAllForwards.cs b/src/LoadTests/ReadAllForwards.cs index 2930b8b38..e8b28a158 100644 --- a/src/LoadTests/ReadAllForwards.cs +++ b/src/LoadTests/ReadAllForwards.cs @@ -28,7 +28,7 @@ public override async Task Run(CancellationToken ct) var stopwatch = Stopwatch.StartNew(); int count = 0; var position = Position.Start; - ReadAllPage page; + IReadAllPage page; do { page = await streamStore.ReadAllForwards(position, diff --git a/src/LoadTests/TestGaps.cs b/src/LoadTests/TestGaps.cs index ee673fb96..007ee9151 100644 --- a/src/LoadTests/TestGaps.cs +++ b/src/LoadTests/TestGaps.cs @@ -65,7 +65,7 @@ public override async Task Run(CancellationToken ct) } } - private static async Task WriteActualGaps(CancellationToken ct, IStreamStore streamStore) + private static async Task WriteActualGaps(CancellationToken ct, IStreamStore streamStore) { var stopwatch = Stopwatch.StartNew(); var count = 0; @@ -83,7 +83,7 @@ private static async Task WriteActualGaps(CancellationToken ct, IStreamStore str } while(!page.IsEnd) { - page = await page.ReadNext(ct); + page = await streamStore.ReadAllForwards(page.NextPosition, 73, false, ct); count += page.Messages.Length; for(int i = 0; i < page.Messages.Length; i++) { @@ -101,7 +101,7 @@ private static async Task WriteActualGaps(CancellationToken ct, IStreamStore str } - private static async Task RunRead(CancellationToken ct, IStreamStore streamStore, int readPageSize) + private static async Task RunRead(CancellationToken ct, IStreamStore streamStore, int readPageSize) { int count = 0; var stopwatch = Stopwatch.StartNew(); diff --git a/src/SqlStreamStore.HAL.DevServer/SqlStreamStore.HAL.DevServer.csproj b/src/SqlStreamStore.HAL.DevServer/SqlStreamStore.HAL.DevServer.csproj index 17c615302..9a2441941 100644 --- a/src/SqlStreamStore.HAL.DevServer/SqlStreamStore.HAL.DevServer.csproj +++ b/src/SqlStreamStore.HAL.DevServer/SqlStreamStore.HAL.DevServer.csproj @@ -1,7 +1,7 @@ Exe - netcoreapp2.1 + net5.0 true false diff --git a/src/SqlStreamStore.HAL.DevServer/SqlStreamStoreFactory.cs b/src/SqlStreamStore.HAL.DevServer/SqlStreamStoreFactory.cs index 0b0e3317e..d893b7cfd 100644 --- a/src/SqlStreamStore.HAL.DevServer/SqlStreamStoreFactory.cs +++ b/src/SqlStreamStore.HAL.DevServer/SqlStreamStoreFactory.cs @@ -7,9 +7,11 @@ namespace SqlStreamStore.HAL.DevServer using Microsoft.Data.SqlClient; using Npgsql; using Serilog; + using SqlStreamStore.Streams; + internal static class SqlStreamStoreFactory { - private delegate Task CreateStreamStore( + private delegate Task> CreateStreamStore( string connectionString, string schema, CancellationToken cancellationToken); @@ -26,7 +28,7 @@ private static readonly IDictionary s_factories = new Dictionary { [inmemory] = CreateInMemoryStreamStore, - [postgres] = CreatePostgresStreamStore, + //[postgres] = CreatePostgresStreamStore, [mssql] = CreateMssqlStreamStore }; @@ -48,13 +50,13 @@ public static Task> Create(CancellationToken cancellat return factory(connectionString, schema, cancellationToken); } - private static Task CreateInMemoryStreamStore( + private static Task> CreateInMemoryStreamStore( string connectionString, string schema, CancellationToken cancellationToken) - => Task.FromResult(new InMemoryStreamStore()); + => Task.FromResult>(new InMemoryStreamStore()); - private static async Task CreateMssqlStreamStore( + private static async Task> CreateMssqlStreamStore( string connectionString, string schema, CancellationToken cancellationToken) @@ -94,7 +96,7 @@ CREATE DATABASE [{connectionStringBuilder.InitialCatalog}] return streamStore; } - private static async Task CreatePostgresStreamStore( + private static async Task> CreatePostgresStreamStore( string connectionString, string schema, CancellationToken cancellationToken) diff --git a/src/SqlStreamStore.HAL/SqlStreamStore.HAL.csproj b/src/SqlStreamStore.HAL/SqlStreamStore.HAL.csproj index c45a33b19..594e4ddb5 100644 --- a/src/SqlStreamStore.HAL/SqlStreamStore.HAL.csproj +++ b/src/SqlStreamStore.HAL/SqlStreamStore.HAL.csproj @@ -1,6 +1,6 @@  - netstandard2.0 + net5.0 $(DefineConstants);LIBLOG_PORTABLE HTTP Server for SQL Stream Store Stream Store - HAL Server diff --git a/src/SqlStreamStore.HAL/SqlStreamStoreHalMiddleware.cs b/src/SqlStreamStore.HAL/SqlStreamStoreHalMiddleware.cs index 45882e381..0bccf6bd1 100644 --- a/src/SqlStreamStore.HAL/SqlStreamStoreHalMiddleware.cs +++ b/src/SqlStreamStore.HAL/SqlStreamStoreHalMiddleware.cs @@ -54,7 +54,7 @@ public static class SqlStreamStoreHalMiddleware public static IApplicationBuilder UseSqlStreamStoreHal( this IApplicationBuilder builder, IStreamStore streamStore, - SqlStreamStoreMiddlewareOptions options = default) + SqlStreamStoreMiddlewareOptions options = default) { if(builder == null) throw new ArgumentNullException(nameof(builder)); diff --git a/src/SqlStreamStore.Http/SqlStreamStore.Http.csproj b/src/SqlStreamStore.Http/SqlStreamStore.Http.csproj index 6b58d086f..c4cd746a8 100644 --- a/src/SqlStreamStore.Http/SqlStreamStore.Http.csproj +++ b/src/SqlStreamStore.Http/SqlStreamStore.Http.csproj @@ -2,7 +2,7 @@ Http (HAL) provider for SQL Stream Store SQL Stream Store - Http (HAL) Provider - netstandard2.0 + net5.0 true SqlStreamStore.Http SqlStreamStore.Http diff --git a/src/SqlStreamStore.MsSql/SqlStreamStore.MsSql.csproj b/src/SqlStreamStore.MsSql/SqlStreamStore.MsSql.csproj index 3c7d47698..30fcd3c0e 100644 --- a/src/SqlStreamStore.MsSql/SqlStreamStore.MsSql.csproj +++ b/src/SqlStreamStore.MsSql/SqlStreamStore.MsSql.csproj @@ -2,7 +2,7 @@ SQL Server (2012+) provider for SQL Stream Store SQL Stream Store - SQL Server (2012+) Provider - netstandard2.0 + net5.0 true SqlStreamStore.MsSql SqlStreamStore.MsSql diff --git a/src/SqlStreamStore.MySql/SqlStreamStore.MySql.csproj b/src/SqlStreamStore.MySql/SqlStreamStore.MySql.csproj index 4328836b4..c3651bc4a 100644 --- a/src/SqlStreamStore.MySql/SqlStreamStore.MySql.csproj +++ b/src/SqlStreamStore.MySql/SqlStreamStore.MySql.csproj @@ -3,7 +3,7 @@ João P. Bragança, Damian Hickey MySQL provider for SQL StreamStore Stream Store - MySQL - netstandard2.0 + net5.0 SqlStreamStore.MySql SqlStreamStore.MySql SqlStreamStore diff --git a/src/SqlStreamStore.Postgres/PostgresStreamStore.ReadAll.cs b/src/SqlStreamStore.Postgres/PostgresStreamStore.ReadAll.cs index a36c1db02..ce9d3f7b4 100644 --- a/src/SqlStreamStore.Postgres/PostgresStreamStore.ReadAll.cs +++ b/src/SqlStreamStore.Postgres/PostgresStreamStore.ReadAll.cs @@ -22,9 +22,8 @@ public PostgresReadAllPage( long nextPosition, bool isEnd, ReadDirection direction, - ReadNextAllPage readNext, StreamMessage[] messages = null, - string transactionSnapshot = null) : base(fromPosition, nextPosition, isEnd, direction, readNext, messages) + string transactionSnapshot = null) : base(fromPosition, nextPosition, isEnd, direction, messages) { TransactionIds = ParseTransactionSnapshot(transactionSnapshot); } @@ -51,7 +50,6 @@ protected override async Task ReadAllForwardsInternal( long fromPositionExclusive, int maxCount, bool prefetch, - ReadNextAllPage readNext, CancellationToken cancellationToken, long fromMaxPositionInclusive = -1) { @@ -93,7 +91,6 @@ protected override async Task ReadAllForwardsInternal( fromPositionExclusive, true, ReadDirection.Forward, - readNext, Array.Empty()); } @@ -138,7 +135,6 @@ protected override async Task ReadAllForwardsInternal( nextPosition, isEnd, ReadDirection.Forward, - readNext, filteredMessages.ToArray(), transactionSnapshot); } @@ -149,7 +145,6 @@ protected override async Task ReadAllBackwardsInternal( long fromPositionExclusive, int maxCount, bool prefetch, - ReadNextAllPage readNext, CancellationToken cancellationToken) { maxCount = maxCount == int.MaxValue ? maxCount - 1 : maxCount; @@ -195,7 +190,6 @@ protected override async Task ReadAllBackwardsInternal( Position.Start, true, ReadDirection.Backward, - readNext, Array.Empty()); } @@ -229,7 +223,6 @@ protected override async Task ReadAllBackwardsInternal( nextPosition, isEnd, ReadDirection.Backward, - readNext, filteredMessages.ToArray()); } } diff --git a/src/SqlStreamStore.Postgres/PostgresStreamStore.cs b/src/SqlStreamStore.Postgres/PostgresStreamStore.cs index fc2ddb29d..30964d34b 100644 --- a/src/SqlStreamStore.Postgres/PostgresStreamStore.cs +++ b/src/SqlStreamStore.Postgres/PostgresStreamStore.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; using System.Data; + using System.Linq; using System.Threading; using System.Threading.Tasks; using Npgsql; @@ -260,6 +261,62 @@ public string GetSchemaCreationScript() return _schema.Definition; } + //protected override async Task HandleGap( + // PostgresReadAllPage page, + // long fromPositionInclusive, + // int maxCount, + // bool prefetchJsonData, + // CancellationToken cancellationToken) + //{ + // if (page.Messages.Length == 0 || DateTime.UtcNow - page.Messages[page.Messages.Length - 1].CreatedUtc > TimeSpan.FromMinutes(5)) + // return page; + + // // TODO: FIXIT + // // Check for gap between last page and this. + // if (page.Messages[0].Position != fromPositionInclusive) + // { + // Logger.DebugFormat("Gap detected at lower page boundary. Potentially could have lost {lostMessageCount} events if the gap is transient", page.Messages[0].Position - fromPositionInclusive); + // page = await HandleGap(page, fromPositionInclusive, maxCount, prefetchJsonData, cancellationToken); + // //if (!page.IsEnd || page.Messages.Length == 1) + // // Logger.DebugFormat("Gap detected at lower page boundary. Potentially could have lost {lostMessageCount} events if the gap is transient", page.Messages[0].Position - fromPositionInclusive); + // //page = await ReloadAfterDelay(fromPositionInclusive, maxCount, prefetchJsonData, ReadNext, cancellationToken); + // } + + // // check for gap in messages collection + // for (int i = 0; i < page.Messages.Length - 1; i++) + // { + // var expectedNextPosition = page.Messages[i].Position + 1; + // if (expectedNextPosition != page.Messages[i + 1].Position) + // { + // Logger.InfoFormat("Gap detected in " + (page.IsEnd ? "last" : "(NOT the last)") + " page. Returning partial page {fromPosition}-{toPosition}", fromPositionInclusive, fromPositionInclusive + i + 1); + + // PostgresReadAllPage requeryPage; + // var maxPosition = page.Messages[page.Messages.Length - 1].Position; + // do + // { + // requeryPage = await ReadAllForwardsInternal(fromPositionInclusive, maxCount, prefetchJsonData, cancellationToken, maxPosition); + // } while (page.TransactionIds.Intersect(requeryPage.TransactionIds).Any()); + + // return requeryPage; + + // // switched this to return the partial page, then re-issue load starting at gap + // // this speeds up the retry instead of taking a 3 second delay immediately + // //var messagesBeforeGap = new StreamMessage[i+1]; + // //page.Messages.Take(i+1).ToArray().CopyTo(messagesBeforeGap, 0); + // //return new ReadAllPage(page.FromPosition, maxPosition, page.IsEnd, page.Direction, ReadNext, messagesBeforeGap); + // } + // } + + // //ReadAllPage requeryPage; + // //var maxPosition = pageWithGap.Messages[pageWithGap.Messages.Length - 1].Position; + // //do + // //{ + // // requeryPage = await ReadAllForwardsInternal(fromPositionInclusive, maxCount, prefetchJsonData, readNext, cancellationToken, maxPosition); + // //} while (pageWithGap.TxSnapshot.CurrentTxIds.Intersect(requeryPage.TxSnapshot.CurrentTxIds).Any()); + + // return page; + //} + //protected override async Task HandleGap(T page, long fromPositionInclusive, int maxCount, bool prefetchJsonData, ReadNextAllPage readNext, CancellationToken cancellationToken) where T : PostgresReadAllPage //{ // if (page.Messages.Length == 0 || DateTime.UtcNow - page.Messages[page.Messages.Length - 1].CreatedUtc > TimeSpan.FromMinutes(5)) diff --git a/src/SqlStreamStore.Postgres/SqlStreamStore.Postgres.csproj b/src/SqlStreamStore.Postgres/SqlStreamStore.Postgres.csproj index 9702d8626..f6cff4000 100644 --- a/src/SqlStreamStore.Postgres/SqlStreamStore.Postgres.csproj +++ b/src/SqlStreamStore.Postgres/SqlStreamStore.Postgres.csproj @@ -2,7 +2,7 @@ Postgre SQL provider for SQL StreamStore Stream Store - Postgres - netstandard2.0 + net5.0 SqlStreamStore.Postgres SqlStreamStore.Postgres SqlStreamStore diff --git a/src/SqlStreamStore.SchemaCreationScriptTool/SqlStreamStore.SchemaCreationScriptTool.csproj b/src/SqlStreamStore.SchemaCreationScriptTool/SqlStreamStore.SchemaCreationScriptTool.csproj index 3381c6f9f..afb6e66d8 100644 --- a/src/SqlStreamStore.SchemaCreationScriptTool/SqlStreamStore.SchemaCreationScriptTool.csproj +++ b/src/SqlStreamStore.SchemaCreationScriptTool/SqlStreamStore.SchemaCreationScriptTool.csproj @@ -4,7 +4,7 @@ A tool to generate the schema creation script of the various SQL based implementations. SQL Stream Store - Schema Creation Script Tool Exe - netcoreapp2.1 + net5.0 true SqlStreamStore.SchemaCreationScriptTool SqlStreamStore.SchemaCreationScriptTool diff --git a/src/SqlStreamStore.Sqlite/SqlStreamStore.Sqlite.csproj b/src/SqlStreamStore.Sqlite/SqlStreamStore.Sqlite.csproj index f25a980ec..d9bfe8cb5 100644 --- a/src/SqlStreamStore.Sqlite/SqlStreamStore.Sqlite.csproj +++ b/src/SqlStreamStore.Sqlite/SqlStreamStore.Sqlite.csproj @@ -2,7 +2,7 @@ Sqlite provider for SQL Stream Store SQL Stream Store - Sqlite Provider - netstandard2.0 + net5.0 true SqlStreamStore.Sqlite SqlStreamStore.Sqlite diff --git a/src/SqlStreamStore.Sqlite/SqliteStreamStore.ReadAll.cs b/src/SqlStreamStore.Sqlite/SqliteStreamStore.ReadAll.cs index 32d3f04f1..7dbbd1b25 100644 --- a/src/SqlStreamStore.Sqlite/SqliteStreamStore.ReadAll.cs +++ b/src/SqlStreamStore.Sqlite/SqliteStreamStore.ReadAll.cs @@ -12,7 +12,6 @@ protected override async Task ReadAllForwardsInternal( long fromPositionExclusive, int maxCount, bool prefetch, - ReadNextAllPage readNext, CancellationToken cancellationToken, long fromMaxPositionInclusive = -1) { @@ -30,8 +29,7 @@ protected override async Task ReadAllForwardsInternal( Position.Start, Position.Start, true, - ReadDirection.Forward, - readNext); + ReadDirection.Forward); } if(allStreamPosition < fromPositionExclusive) @@ -40,8 +38,7 @@ protected override async Task ReadAllForwardsInternal( fromPositionExclusive, fromPositionExclusive, true, - ReadDirection.Forward, - readNext); + ReadDirection.Forward); } var remaining = await connection.AllStream() @@ -53,8 +50,7 @@ protected override async Task ReadAllForwardsInternal( fromPositionExclusive, Position.End, true, - ReadDirection.Forward, - readNext); + ReadDirection.Forward); } var messages = await connection.AllStream() @@ -75,7 +71,6 @@ protected override async Task ReadAllForwardsInternal( nextPosition, isEnd, ReadDirection.Forward, - readNext, messages.ToArray()); } } @@ -84,7 +79,6 @@ protected override async Task ReadAllBackwardsInternal( long fromPosition, int maxCount, bool prefetch, - ReadNextAllPage readNext, CancellationToken cancellationToken) { GuardAgainstDisposed(); @@ -101,8 +95,7 @@ protected override async Task ReadAllBackwardsInternal( Position.Start, Position.Start, true, - ReadDirection.Backward, - readNext); + ReadDirection.Backward); } if(fromPosition == Position.End) @@ -116,8 +109,7 @@ protected override async Task ReadAllBackwardsInternal( fromPosition, fromPosition, true, - ReadDirection.Backward, - readNext); + ReadDirection.Backward); } // For reading $all, in the case where no events have been entered into @@ -135,8 +127,7 @@ protected override async Task ReadAllBackwardsInternal( allStreamPosition ?? Position.Start, Position.End, true, - ReadDirection.Backward, - readNext); + ReadDirection.Backward); } var messages = await connection.AllStream() @@ -155,7 +146,6 @@ protected override async Task ReadAllBackwardsInternal( nextPosition, isEnd, ReadDirection.Backward, - readNext, messages.ToArray()); } } diff --git a/src/SqlStreamStore.TestUtils/SqlStreamStore.TestUtils.csproj b/src/SqlStreamStore.TestUtils/SqlStreamStore.TestUtils.csproj index 67653edce..96f2ad28c 100644 --- a/src/SqlStreamStore.TestUtils/SqlStreamStore.TestUtils.csproj +++ b/src/SqlStreamStore.TestUtils/SqlStreamStore.TestUtils.csproj @@ -1,6 +1,6 @@  - netcoreapp3.1 + net5.0 SqlStreamStore.TestUtils false diff --git a/src/SqlStreamStore/IReadonlyStreamStore.cs b/src/SqlStreamStore/IReadonlyStreamStore.cs index 853d25d73..a15f30889 100644 --- a/src/SqlStreamStore/IReadonlyStreamStore.cs +++ b/src/SqlStreamStore/IReadonlyStreamStore.cs @@ -3,6 +3,7 @@ using System; using System.Threading; using System.Threading.Tasks; + using MorseCode.ITask; using SqlStreamStore.Streams; using SqlStreamStore.Subscriptions; @@ -251,7 +252,7 @@ Task ListStreams( /// /// Represents a readonly stream store. /// - public interface IReadonlyStreamStore : IReadonlyStreamStore where TReadAllPage : IReadAllPage + public interface IReadonlyStreamStore : IReadonlyStreamStore where TReadAllPage : IReadAllPage { /// /// Reads messages from all streams forwards. @@ -274,7 +275,7 @@ public interface IReadonlyStreamStore : IReadonlyStreamStore where /// An presenting the result of the read. If all messages read have expired /// then the message collection MAY be empty. /// - Task ReadAllForwards( + ITask ReadAllForwards( long fromPositionInclusive, int maxCount, bool prefetchJsonData = true, @@ -301,7 +302,7 @@ Task ReadAllForwards( /// An presenting the result of the read. If all messages read have expired /// then the message collection MAY be empty. /// - Task ReadAllBackwards( + ITask ReadAllBackwards( long fromPositionInclusive, int maxCount, bool prefetchJsonData = true, diff --git a/src/SqlStreamStore/IStreamStore.cs b/src/SqlStreamStore/IStreamStore.cs index a4fb388f5..1d1d7bae6 100644 --- a/src/SqlStreamStore/IStreamStore.cs +++ b/src/SqlStreamStore/IStreamStore.cs @@ -123,7 +123,7 @@ Task SetStreamMetadata( } // TODO - public interface IStreamStore : IStreamStore, IReadonlyStreamStore where TReadPage : IReadAllPage + public interface IStreamStore : IStreamStore, IReadonlyStreamStore where TReadPage : IReadAllPage { } } diff --git a/src/SqlStreamStore/InMemory/InMemoryAllStream.cs b/src/SqlStreamStore/InMemory/InMemoryAllStream.cs new file mode 100644 index 000000000..cf255ab87 --- /dev/null +++ b/src/SqlStreamStore/InMemory/InMemoryAllStream.cs @@ -0,0 +1,7 @@ +namespace SqlStreamStore.InMemory +{ + using System.Collections.Generic; + + internal class InMemoryAllStream : LinkedList + {} +} \ No newline at end of file diff --git a/src/SqlStreamStore/InMemory/InMemoryStream.cs b/src/SqlStreamStore/InMemory/InMemoryStream.cs new file mode 100644 index 000000000..db9e659d3 --- /dev/null +++ b/src/SqlStreamStore/InMemory/InMemoryStream.cs @@ -0,0 +1,225 @@ +namespace SqlStreamStore.InMemory +{ + using System; + using System.Collections.Generic; + using System.Linq; + using SqlStreamStore.Infrastructure; + using SqlStreamStore.Streams; + + internal class InMemoryStream + { + private readonly string _streamId; + private readonly InMemoryAllStream _inMemoryAllStream; + private readonly GetUtcNow _getUtcNow; + private readonly Action _onStreamAppended; + private readonly Func _getNextPosition; + private readonly List _messages = new List(); + private readonly Dictionary _messagesById = new Dictionary(); + + internal InMemoryStream( + string streamId, + InMemoryAllStream inMemoryAllStream, + GetUtcNow getUtcNow, + Action onStreamAppended, + Func getNextPosition) + { + _streamId = streamId; + _inMemoryAllStream = inMemoryAllStream; + _getUtcNow = getUtcNow; + _onStreamAppended = onStreamAppended; + _getNextPosition = getNextPosition; + } + + internal IReadOnlyList Messages => _messages; + + internal int CurrentVersion { get; private set; } = -1; + + internal int CurrentPosition { get; private set; } = -1; + + internal void AppendToStream(int expectedVersion, NewStreamMessage[] newMessages) + { + switch(expectedVersion) + { + case ExpectedVersion.Any: + AppendToStreamExpectedVersionAny(expectedVersion, newMessages); + return; + case ExpectedVersion.NoStream: + AppendToStreamExpectedVersionNoStream(expectedVersion, newMessages); + return; + default: + AppendToStreamExpectedVersion(expectedVersion, newMessages); + return; + } + } + + private void AppendToStreamExpectedVersion(int expectedVersion, NewStreamMessage[] newMessages) + { + // Need to do optimistic concurrency check... + if(expectedVersion > CurrentVersion) + { + throw new WrongExpectedVersionException( + ErrorMessages.AppendFailedWrongExpectedVersion(_streamId, expectedVersion), + _streamId, + expectedVersion); + } + + if(CurrentVersion >= 0 && expectedVersion < CurrentVersion) + { + // expectedVersion < currentVersion, Idempotency test + for(int i = 0; i < newMessages.Length; i++) + { + int index = expectedVersion + i + 1; + if(index >= _messages.Count) + { + throw new WrongExpectedVersionException( + ErrorMessages.AppendFailedWrongExpectedVersion(_streamId, expectedVersion), + _streamId, + expectedVersion); + } + if(_messages[index].MessageId != newMessages[i].MessageId) + { + throw new WrongExpectedVersionException( + ErrorMessages.AppendFailedWrongExpectedVersion(_streamId, expectedVersion), + _streamId, + expectedVersion); + } + } + return; + } + + // expectedVersion == currentVersion) + if(newMessages.Any(newmessage => _messagesById.ContainsKey(newmessage.MessageId))) + { + throw new WrongExpectedVersionException( + ErrorMessages.AppendFailedWrongExpectedVersion(_streamId, expectedVersion), + _streamId, + expectedVersion); + } + + AppendEvents(newMessages); + } + + private void AppendToStreamExpectedVersionAny(int expectedVersion, NewStreamMessage[] newMessages) + { + if(newMessages?.Length > 0) + { + // idemponcy check - have messages already been written? + if(_messagesById.TryGetValue(newMessages[0].MessageId, out var item)) + { + int i = _messages.IndexOf(item); + if(i + newMessages.Length > _messages.Count) + { + throw new WrongExpectedVersionException( + ErrorMessages.AppendFailedWrongExpectedVersion(_streamId, expectedVersion), + _streamId, + expectedVersion); + } + + for(int n = 1; n < newMessages.Length; n++) + { + if(newMessages[n].MessageId != _messages[i + n].MessageId) + { + throw new WrongExpectedVersionException( + ErrorMessages.AppendFailedWrongExpectedVersion(_streamId, expectedVersion), + _streamId, + expectedVersion); + } + } + + return; + } + } + + // None of the Messages were written previously... + AppendEvents(newMessages); + } + + private void AppendToStreamExpectedVersionNoStream(int expectedVersion, NewStreamMessage[] newMessages) + { + if(_messages.Count > 0) + { + //Already committed Messages, do idempotency check + if(newMessages.Length > _messages.Count) + { + throw new WrongExpectedVersionException( + ErrorMessages.AppendFailedWrongExpectedVersion(_streamId, expectedVersion), + _streamId, + expectedVersion); + } + + if(newMessages.Where((message, index) => _messages[index].MessageId != message.MessageId).Any()) + { + throw new WrongExpectedVersionException( + ErrorMessages.AppendFailedWrongExpectedVersion(_streamId, expectedVersion), + _streamId, + expectedVersion); + } + return; + } + + // None of the Messages were written previously... + AppendEvents(newMessages); + } + + private void AppendEvents(NewStreamMessage[] newMessages) + { + foreach(var newmessage in newMessages) + { + var position = _getNextPosition(); + CurrentVersion++; + CurrentPosition = position; + + var inMemorymessage = new InMemoryStreamMessage( + _streamId, + newmessage.MessageId, + CurrentVersion, + position, + _getUtcNow(), + newmessage.Type, + newmessage.JsonData, + newmessage.JsonMetadata); + + _messages.Add(inMemorymessage); + _messagesById.Add(newmessage.MessageId, inMemorymessage); + _inMemoryAllStream.AddAfter(_inMemoryAllStream.Last, inMemorymessage); + } + _onStreamAppended(); + } + + internal void DeleteAllEvents(int expectedVersion) + { + if (expectedVersion > 0 && expectedVersion != CurrentVersion) + { + throw new WrongExpectedVersionException( + ErrorMessages.AppendFailedWrongExpectedVersion(_streamId, expectedVersion), + _streamId, + expectedVersion); + } + + foreach (var inMemorymessage in _messages) + { + _inMemoryAllStream.Remove(inMemorymessage); + } + _messages.Clear(); + _messagesById.Clear(); + } + + internal bool DeleteEvent(Guid eventId) + { + if (!_messagesById.TryGetValue(eventId, out var inMemoryStreamMessage)) + { + return false; + } + + _messages.Remove(inMemoryStreamMessage); + _inMemoryAllStream.Remove(inMemoryStreamMessage); + _messagesById.Remove(eventId); + return true; + } + + internal string GetMessageData(Guid messageId) + { + return _messagesById.TryGetValue(messageId, out var message) ? message.JsonData : string.Empty; + } + } +} \ No newline at end of file diff --git a/src/SqlStreamStore/InMemory/InMemoryStreamMessage.cs b/src/SqlStreamStore/InMemory/InMemoryStreamMessage.cs new file mode 100644 index 000000000..51c59d9c4 --- /dev/null +++ b/src/SqlStreamStore/InMemory/InMemoryStreamMessage.cs @@ -0,0 +1,36 @@ +namespace SqlStreamStore.InMemory +{ + using System; + + internal sealed class InMemoryStreamMessage + { + internal readonly long Position; + internal readonly DateTime Created; + internal readonly string StreamId; + internal readonly Guid MessageId; + internal readonly string JsonData; + internal readonly string JsonMetadata; + internal readonly int StreamVersion; + internal readonly string Type; + + internal InMemoryStreamMessage( + string streamId, + Guid messageId, + int streamVersion, + long position, + DateTime created, + string type, + string jsonData, + string jsonMetadata) + { + StreamId = streamId; + MessageId = messageId; + StreamVersion = streamVersion; + Position = position; + Created = created; + Type = type; + JsonData = jsonData; + JsonMetadata = jsonMetadata; + } + } +} \ No newline at end of file diff --git a/src/SqlStreamStore/InMemory/InMemoryStreamStore.cs b/src/SqlStreamStore/InMemory/InMemoryStreamStore.cs new file mode 100644 index 000000000..80d9f4e5e --- /dev/null +++ b/src/SqlStreamStore/InMemory/InMemoryStreamStore.cs @@ -0,0 +1,778 @@ +// ReSharper disable once CheckNamespace +namespace SqlStreamStore +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using SqlStreamStore.Imports.Ensure.That; + using SqlStreamStore.Infrastructure; + using SqlStreamStore.InMemory; + using SqlStreamStore.Streams; + using SqlStreamStore.Subscriptions; + using StreamStoreStore.Json; + using static Streams.Deleted; + + /// + /// Represents an in-memory implementation of a stream store. Use for testing or high/speed + volatile scenarios. + /// + public sealed class InMemoryStreamStore : StreamStoreBase + { + private readonly InMemoryAllStream _allStream = new InMemoryAllStream(); + private readonly GetUtcNow _getUtcNow; + private readonly ReaderWriterLockSlim _lock = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion); + private readonly Action _onStreamAppended; + private readonly Dictionary _streams = new Dictionary(); + private readonly Subject _subscriptions = new Subject(); + private readonly InterlockedBoolean _signallingToSubscribers = new InterlockedBoolean(); + private readonly IList _streamIds = new List(); + private int _currentPosition; + private static readonly ReadNextStreamPage s_readNextNotFound = + (_, ct) => throw new InvalidOperationException("Cannot read next page of non-exisitent stream"); + + + /// + /// Initializes a new instance of + /// + /// Optional. A delegate to ge the current UTC now. Used to define generate timestamps. + /// Optional. The name of the logger generated log messages will use. + public InMemoryStreamStore(GetUtcNow getUtcNow = null, string logName = null) + : base(TimeSpan.FromMinutes(1), 10000, getUtcNow, logName ?? nameof(InMemoryStreamStore)) + { + _getUtcNow = getUtcNow ?? SystemClock.GetUtcNow; + _allStream.AddFirst(new InMemoryStreamMessage( + "", + Guid.NewGuid(), + -1, + -1, + _getUtcNow(), + null, + null, + null)); + _onStreamAppended = () => + { + if(_signallingToSubscribers.CompareExchange(true, false) == false) + { + Task.Run(() => + { + _subscriptions.OnNext(Unit.Default); + _signallingToSubscribers.Set(false); + }); + } + }; + } + + protected override void Dispose(bool disposing) + { + using(_lock.UseWriteLock()) + { + _subscriptions.OnCompleted(); + _allStream.Clear(); + _streams.Clear(); + } + } + + private Task GetStreamMessageCount(string streamId, CancellationToken cancellationToken) + { + using(_lock.UseReadLock()) + { + return Task.FromResult(!_streams.ContainsKey(streamId) ? 0 : _streams[streamId].Messages.Count); + } + } + + protected override async Task AppendToStreamInternal( + string streamId, + int expectedVersion, + NewStreamMessage[] messages, + CancellationToken cancellationToken) + { + GuardAgainstDisposed(); + cancellationToken.ThrowIfCancellationRequested(); + + AppendResult appendResult; + using (_lock.UseWriteLock()) + { + appendResult = AppendToStreamInternal(streamId, expectedVersion, messages); + } + var meta = await GetStreamMetadataInternal(streamId, cancellationToken); + await CheckStreamMaxCount(streamId, meta.MaxCount, cancellationToken); + return appendResult; + } + + private async Task CheckStreamMaxCount(string streamId, int? maxCount, CancellationToken cancellationToken) + { + if (maxCount.HasValue) + { + var count = await GetStreamMessageCount(streamId, cancellationToken); + if (count > maxCount.Value) + { + int toPurge = count - maxCount.Value; + + var streamMessagesPage = await ReadStreamForwardsInternal(streamId, StreamVersion.Start, + toPurge, false, null, cancellationToken); + + if (streamMessagesPage.Status == PageReadStatus.Success) + { + foreach (var message in streamMessagesPage.Messages) + { + await DeleteEventInternal(streamId, message.MessageId, cancellationToken); + } + } + } + } + } + + private AppendResult AppendToStreamInternal( + string streamId, + int expectedVersion, + NewStreamMessage[] messages) + { + InMemoryStream inMemoryStream; + if (expectedVersion == ExpectedVersion.NoStream || expectedVersion == ExpectedVersion.Any) + { + if (_streams.TryGetValue(streamId, out inMemoryStream)) + { + inMemoryStream.AppendToStream(expectedVersion, messages); + } + else + { + inMemoryStream = new InMemoryStream( + streamId, + _allStream, + _getUtcNow, + _onStreamAppended, + () => _currentPosition++); + inMemoryStream.AppendToStream(expectedVersion, messages); + _streams.Add(streamId, inMemoryStream); + _streamIds.Add(streamId); + } + return new AppendResult(inMemoryStream.CurrentVersion, inMemoryStream.CurrentPosition); + } + + if (!_streams.TryGetValue(streamId, out inMemoryStream)) + { + throw new WrongExpectedVersionException( + ErrorMessages.AppendFailedWrongExpectedVersion(streamId, expectedVersion), + streamId, + expectedVersion); + } + inMemoryStream.AppendToStream(expectedVersion, messages); + return new AppendResult(inMemoryStream.CurrentVersion, inMemoryStream.CurrentPosition); + } + + protected override Task DeleteEventInternal(string streamId, Guid eventId, CancellationToken cancellationToken) + { + GuardAgainstDisposed(); + cancellationToken.ThrowIfCancellationRequested(); + + using (_lock.UseWriteLock()) + { + if (!_streams.ContainsKey(streamId)) + { + return Task.FromResult(0); + } + + var inMemoryStream = _streams[streamId]; + bool deleted = inMemoryStream.DeleteEvent(eventId); + + if (deleted) + { + var eventDeletedEvent = CreateMessageDeletedMessage(streamId, eventId); + AppendToStreamInternal(DeletedStreamId, ExpectedVersion.Any, new[] { eventDeletedEvent }); + } + + return Task.FromResult(0); + } + } + + protected override async Task GetStreamMetadataInternal( + string streamId, + CancellationToken cancellationToken) + { + GuardAgainstDisposed(); + cancellationToken.ThrowIfCancellationRequested(); + + using (_lock.UseReadLock()) + { + string metaStreamId = $"$${streamId}"; + + var eventsPage = await ReadStreamBackwardsInternal(metaStreamId, StreamVersion.End, + 1, true, null, cancellationToken); + + if (eventsPage.Status == PageReadStatus.StreamNotFound) + { + return new StreamMetadataResult(streamId, -1); + } + + var metadataMessage = await eventsPage.Messages[0].GetJsonDataAs(cancellationToken); + + return new StreamMetadataResult( + streamId, + eventsPage.LastStreamVersion, + metadataMessage.MaxAge, + metadataMessage.MaxCount, + metadataMessage.MetaJson); + } + } + + protected override async Task SetStreamMetadataInternal( + string streamId, + int expectedStreamMetadataVersion, + int? maxAge, + int? maxCount, + string metadataJson, + CancellationToken cancellationToken) + { + using(_lock.UseWriteLock()) + { + string metaStreamId = $"$${streamId}"; + + var metadataMessage = new MetadataMessage + { + StreamId = streamId, + MaxAge = maxAge, + MaxCount = maxCount, + MetaJson = metadataJson + }; + var json = SimpleJson.SerializeObject(metadataMessage); + var messageId = MetadataMessageIdGenerator.Create(metaStreamId, expectedStreamMetadataVersion, json); + var newStreamMessage = new NewStreamMessage(messageId, "$stream-metadata", json); + + var result = AppendToStreamInternal(metaStreamId, expectedStreamMetadataVersion, new[] { newStreamMessage }); + + await CheckStreamMaxCount(streamId, metadataMessage.MaxCount, cancellationToken); + + return new SetStreamMetadataResult(result.CurrentVersion); + } + } + + protected override Task DeleteStreamInternal( + string streamId, + int expectedVersion, + CancellationToken cancellationToken) + { + GuardAgainstDisposed(); + + using(_lock.UseWriteLock()) + { + DeleteStream(streamId, expectedVersion); + + // Delete metadata stream, if it exists + DeleteStream($"$${streamId}", ExpectedVersion.Any); + + return Task.FromResult(0); + } + } + + private void DeleteStream(string streamId, int expectedVersion) + { + if (!_streams.ContainsKey(streamId)) + { + if (expectedVersion >= 0) + { + throw new WrongExpectedVersionException( + ErrorMessages.AppendFailedWrongExpectedVersion(streamId, expectedVersion), + streamId, + expectedVersion); + } + return; + } + if (expectedVersion != ExpectedVersion.Any && + _streams[streamId].Messages.Last().StreamVersion != expectedVersion) + { + throw new WrongExpectedVersionException( + ErrorMessages.AppendFailedWrongExpectedVersion(streamId, expectedVersion), + streamId, + expectedVersion); + } + InMemoryStream inMemoryStream = _streams[streamId]; + _streams.Remove(streamId); + inMemoryStream.DeleteAllEvents(ExpectedVersion.Any); + _streamIds[_streamIds.IndexOf(streamId)] = null; + + var streamDeletedEvent = CreateStreamDeletedMessage(streamId); + AppendToStreamInternal(DeletedStreamId, ExpectedVersion.Any, new[] { streamDeletedEvent }); + } + + protected override Task ReadAllForwardsInternal(long fromPositionExclusive, int maxCount, + bool prefetch, CancellationToken cancellationToken, long fromMaxPositionInclusive = -1) + { + GuardAgainstDisposed(); + cancellationToken.ThrowIfCancellationRequested(); + + using (_lock.UseReadLock()) + { + // Find the node to start from (it may not be equal to the exact position) + var current = _allStream.First; + if(current.Next == null) //Empty store + { + var result = new ReadAllPage(Position.Start, Position.Start, true, ReadDirection.Forward); + return Task.FromResult(result); + } + + var previous = current.Previous; + while(current.Value.Position < fromPositionExclusive) + { + if(current.Next == null) // fromPosition is past end of store + { + var result = new ReadAllPage(fromPositionExclusive, fromPositionExclusive, true, + ReadDirection.Forward); + return Task.FromResult(result); + } + previous = current; + current = current.Next; + } + + var messages = new List(); + while(maxCount > 0 && current != null) + { + StreamMessage message; + if (prefetch) + { + message = new StreamMessage( + current.Value.StreamId, + current.Value.MessageId, + current.Value.StreamVersion, + current.Value.Position, + current.Value.Created, + current.Value.Type, + current.Value.JsonMetadata, + current.Value.JsonData); + } + else + { + var currentCopy = current; + message = new StreamMessage( + current.Value.StreamId, + current.Value.MessageId, + current.Value.StreamVersion, + current.Value.Position, + current.Value.Created, + current.Value.Type, + current.Value.JsonMetadata, + ct => + { + return Task.Run( + () => ReadMessageData(currentCopy.Value.StreamId, currentCopy.Value.MessageId), ct); + }); + } + messages.Add(message); + maxCount--; + previous = current; + current = current.Next; + } + + var isEnd = current == null; + var nextPosition = current?.Value.Position ?? previous.Value.Position + 1; + fromPositionExclusive = messages.Any() ? messages[0].Position : 0; + + var page = new ReadAllPage( + fromPositionExclusive, + nextPosition, + isEnd, + ReadDirection.Forward, + messages.ToArray()); + + return Task.FromResult(page); + } + } + + protected override Task ReadAllBackwardsInternal( + long fromPositionExclusive, + int maxCount, + bool prefetch, + CancellationToken cancellationToken) + { + GuardAgainstDisposed(); + + using (_lock.UseReadLock()) + { + if (fromPositionExclusive == Position.End) + { + fromPositionExclusive = _allStream.Last.Value.Position; + } + + // Find the node to start from (it may not be equal to the exact position) + var current = _allStream.First; + if(current.Next == null) //Empty store + { + var result = new ReadAllPage(Position.Start, Position.Start, true, ReadDirection.Backward); + return Task.FromResult(result); + } + + var previous = current.Previous; + while(current.Value.Position < fromPositionExclusive) + { + if(current.Next == null) // fromPosition is past end of store + { + var result = new ReadAllPage(fromPositionExclusive, fromPositionExclusive, true, + ReadDirection.Backward); + return Task.FromResult(result); + } + previous = current; + current = current.Next; + } + + var messages = new List(); + while(maxCount > 0 && current != _allStream.First) + { + StreamMessage message; + if (prefetch) + { + message = new StreamMessage( + current.Value.StreamId, + current.Value.MessageId, + current.Value.StreamVersion, + current.Value.Position, + current.Value.Created, + current.Value.Type, + current.Value.JsonMetadata, + current.Value.JsonData); + } + else + { + var currentCopy = current; + message = new StreamMessage( + current.Value.StreamId, + current.Value.MessageId, + current.Value.StreamVersion, + current.Value.Position, + current.Value.Created, + current.Value.Type, + current.Value.JsonMetadata, + ct => + { + return Task.Run( + () => ReadMessageData(currentCopy.Value.StreamId, currentCopy.Value.MessageId), ct); + }); + } + messages.Add(message); + + maxCount--; + previous = current; + current = current.Previous; + } + + bool isEnd; + if(previous == null || previous.Value.Position == 0) + { + isEnd = true; + } + else + { + isEnd = false; + } + var nextPosition = isEnd + ? 0 + : current.Value.Position; + + fromPositionExclusive = messages.Any() ? messages[0].Position : 0; + + var page = new ReadAllPage( + fromPositionExclusive, + nextPosition, + isEnd, + ReadDirection.Backward, + messages.ToArray()); + + return Task.FromResult(page); + } + } + + protected override Task ReadStreamForwardsInternal(string streamId, int start, int count, bool prefetch, ReadNextStreamPage readNext, CancellationToken cancellationToken) + { + GuardAgainstDisposed(); + cancellationToken.ThrowIfCancellationRequested(); + + using (_lock.UseReadLock()) + { + if(!_streams.TryGetValue(streamId, out var stream)) + { + var notFound = new ReadStreamPage( + streamId, + PageReadStatus.StreamNotFound, + start, + -1, + -1, + -1, + ReadDirection.Forward, + true, + s_readNextNotFound); + return Task.FromResult(notFound); + } + + var messages = new List(); + var i = start; + + while(i < stream.Messages.Count && count > 0) + { + var inMemorymessage = stream.Messages[i]; + StreamMessage message; + if (prefetch) + { + message = new StreamMessage( + streamId, + inMemorymessage.MessageId, + inMemorymessage.StreamVersion, + inMemorymessage.Position, + inMemorymessage.Created, + inMemorymessage.Type, + inMemorymessage.JsonMetadata, + inMemorymessage.JsonData); + } + else + { + message = new StreamMessage( + streamId, + inMemorymessage.MessageId, + inMemorymessage.StreamVersion, + inMemorymessage.Position, + inMemorymessage.Created, + inMemorymessage.Type, + inMemorymessage.JsonMetadata, + ct => Task.Run(() => ReadMessageData(streamId, inMemorymessage.MessageId), ct)); + } + messages.Add(message); + + i++; + count--; + } + + var lastStreamVersion = stream.CurrentVersion; + int nextStreamVersion; + if(lastStreamVersion == -1) + { + nextStreamVersion = 0; + } + else if(messages.Count == 0) + { + nextStreamVersion = lastStreamVersion + 1; + } + else + { + nextStreamVersion = messages.Last().StreamVersion + 1; + } + var endOfStream = i == stream.Messages.Count; + + var page = new ReadStreamPage( + streamId, + PageReadStatus.Success, + start, + nextStreamVersion, + lastStreamVersion, + stream.CurrentPosition, + ReadDirection.Forward, + endOfStream, + readNext, + messages.ToArray()); + + return Task.FromResult(page); + } + } + + protected override Task ReadStreamBackwardsInternal( + string streamId, + int fromVersionInclusive, + int count, + bool prefetch, + ReadNextStreamPage readNext, + CancellationToken cancellationToken) + { + GuardAgainstDisposed(); + cancellationToken.ThrowIfCancellationRequested(); + + using (_lock.UseReadLock()) + { + if (!_streams.TryGetValue(streamId, out var stream)) + { + var notFound = new ReadStreamPage(streamId, + PageReadStatus.StreamNotFound, + fromVersionInclusive, + -1, + -1, + -1, + ReadDirection.Backward, + true, + s_readNextNotFound); + return Task.FromResult(notFound); + } + + var messages = new List(); + var i = fromVersionInclusive == StreamVersion.End ? stream.Messages.Count - 1 : fromVersionInclusive; + while (i < stream.Messages.Count && i >= 0 && count > 0) + { + var inMemorymessage = stream.Messages[i]; + StreamMessage message; + if (prefetch) + { + message = new StreamMessage( + streamId, + inMemorymessage.MessageId, + inMemorymessage.StreamVersion, + inMemorymessage.Position, + inMemorymessage.Created, + inMemorymessage.Type, + inMemorymessage.JsonMetadata, + inMemorymessage.JsonData); + } + else + { + message = new StreamMessage( + streamId, + inMemorymessage.MessageId, + inMemorymessage.StreamVersion, + inMemorymessage.Position, + inMemorymessage.Created, + inMemorymessage.Type, + inMemorymessage.JsonMetadata, + ct => Task.Run(() => ReadMessageData(streamId, inMemorymessage.MessageId), ct)); + } + messages.Add(message); + + i--; + count--; + } + + var lastStreamVersion = stream.Messages.Count > 0 + ? stream.Messages[stream.Messages.Count - 1].StreamVersion + : StreamVersion.End; + var nextStreamVersion = messages.Count > 0 + ? messages[messages.Count - 1].StreamVersion - 1 + : StreamVersion.End; + var endOfStream = nextStreamVersion < 0; + + var page = new ReadStreamPage( + streamId, + PageReadStatus.Success, + fromVersionInclusive, + nextStreamVersion, + lastStreamVersion, + stream.CurrentPosition, + ReadDirection.Backward, + endOfStream, + readNext, + messages.ToArray()); + + return Task.FromResult(page); + } + } + + private string ReadMessageData(string streamId, Guid messageId) + { + using(_lock.UseReadLock()) + { + InMemoryStream stream; + return !_streams.TryGetValue(streamId, out stream) ? null : stream.GetMessageData(messageId); + } + } + + protected override IStreamSubscription SubscribeToStreamInternal( + string streamId, + int? startVersion, + StreamMessageReceived streamMessageReceived, + SubscriptionDropped subscriptionDropped, + HasCaughtUp hasCaughtUp, + bool prefetchJsonData, + string name) + { + return new StreamSubscription( + streamId, + startVersion, + this, + _subscriptions, + streamMessageReceived, + subscriptionDropped, + hasCaughtUp, + prefetchJsonData, + name); + } + + protected override Task ReadHeadPositionInternal(CancellationToken cancellationToken) + { + var message = _allStream.LastOrDefault(); + return message == null ? Task.FromResult(-1L) : Task.FromResult(message.Position); + } + + protected override Task ReadStreamHeadPositionInternal(string streamId, CancellationToken cancellationToken) + { + using(_lock.UseReadLock()) + { + return !_streams.TryGetValue(streamId, out InMemoryStream stream) ? Task.FromResult(-1L) : Task.FromResult(Convert.ToInt64(stream.CurrentPosition)); + } + } + + protected override Task ReadStreamHeadVersionInternal(string streamId, CancellationToken cancellationToken) + { + using(_lock.UseReadLock()) + { + return !_streams.TryGetValue(streamId, out InMemoryStream stream) ? Task.FromResult(-1) : Task.FromResult(stream.CurrentVersion); + } + } + + protected override IAllStreamSubscription SubscribeToAllInternal( + long? fromPosition, + AllStreamMessageReceived streamMessageReceived, + AllSubscriptionDropped subscriptionDropped, + HasCaughtUp hasCaughtUp, + bool prefetchJsonData, + string name) + { + return new AllStreamSubscription( + fromPosition, + this, + _subscriptions, + streamMessageReceived, + subscriptionDropped, + hasCaughtUp, + prefetchJsonData, + name); + } + + protected override Task ListStreamsInternal( + Pattern pattern, + int maxCount, + string continuationToken, + ListNextStreamsPage listNextStreamsPage, + CancellationToken cancellationToken) + { + Ensure.That(listNextStreamsPage).IsNotNull(); + int.TryParse(continuationToken, out var index); + + Func filter; + + switch(pattern) + { + case Pattern.Any _: + filter = s => s != default; + break; + case Pattern.StartingWith p: + filter = s => s != default && s.StartsWith(p.Value); + break; + case Pattern.EndingWith p: + filter = s => s != default && s.EndsWith(p.Value); + break; + default: + throw Pattern.Unrecognized(nameof(pattern)); + } + + using(_lock.UseReadLock()) + { + var streamIds = _streamIds + .Skip(index) + .Where(filter) + .Take(maxCount) + .ToArray(); + + var nextContinuationToken = streamIds.Length == 0 + ? 0 + : _streamIds.IndexOf(streamIds[streamIds.Length - 1]) + 1; + + return Task.FromResult(new ListStreamsPage( + nextContinuationToken.ToString(), + streamIds, + listNextStreamsPage)); + } + } + } +} \ No newline at end of file diff --git a/src/SqlStreamStore/Infrastructure/ReadonlyStreamStoreBase.cs b/src/SqlStreamStore/Infrastructure/ReadonlyStreamStoreBase.cs index d58777fcc..3e0be1a87 100644 --- a/src/SqlStreamStore/Infrastructure/ReadonlyStreamStoreBase.cs +++ b/src/SqlStreamStore/Infrastructure/ReadonlyStreamStoreBase.cs @@ -4,6 +4,7 @@ namespace SqlStreamStore.Infrastructure using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; + using MorseCode.ITask; using SqlStreamStore.Logging; using SqlStreamStore.Streams; using SqlStreamStore.Subscriptions; @@ -42,7 +43,7 @@ protected ReadonlyStreamStoreBase(GetUtcNow getUtcNow, string logName) _disableMetadataCache = true; } - public async Task ReadAllForwards( + public async ITask ReadAllForwards( long fromPositionInclusive, int maxCount, bool prefetchJsonData, @@ -57,14 +58,14 @@ public async Task ReadAllForwards( Logger.DebugFormat("ReadAllForwards from position {fromPositionInclusive} with max count " + "{maxCount}.", fromPositionInclusive, maxCount); - Task ReadNext(long nextPosition, CancellationToken ct) => ReadAllForwards(nextPosition, maxCount, prefetchJsonData, ct); + //Task ReadNext(long nextPosition, CancellationToken ct) => ReadAllForwards(nextPosition, maxCount, prefetchJsonData, ct); - var page = await ReadAllForwardsInternal(fromPositionInclusive, maxCount, prefetchJsonData, ReadNext, cancellationToken) + var page = await ReadAllForwardsInternal(fromPositionInclusive, maxCount, prefetchJsonData, cancellationToken) .ConfigureAwait(false); - var pageHandled = await HandleGap(page, fromPositionInclusive, maxCount, prefetchJsonData, ReadNext, cancellationToken); + var pageHandled = await HandleGap(page, fromPositionInclusive, maxCount, prefetchJsonData, cancellationToken); - return await FilterExpired(pageHandled, ReadNext, cancellationToken).ConfigureAwait(false); + return await FilterExpired(pageHandled, cancellationToken).ConfigureAwait(false); //// https://github.com/damianh/SqlStreamStore/issues/31 //// Under heavy parallel load, gaps may appear in the position sequence due to sequence @@ -173,7 +174,7 @@ public async Task ReadAllForwards( // return page; //} - protected virtual async Task HandleGap(TReadAllPage page, long fromPositionInclusive, int maxCount, bool prefetchJsonData, ReadNextAllPage readNext, CancellationToken cancellationToken) + protected virtual async Task HandleGap(TReadAllPage page, long fromPositionInclusive, int maxCount, bool prefetchJsonData, CancellationToken cancellationToken) { // https://github.com/damianh/SqlStreamStore/issues/31 // Under heavy parallel load, gaps may appear in the position sequence due to sequence @@ -188,7 +189,7 @@ protected virtual async Task HandleGap(TReadAllPage page, long fro // Check for gap between last page and this. if (page.Messages[0].Position != fromPositionInclusive) { - return await ReloadAfterDelay(fromPositionInclusive, maxCount, prefetchJsonData, readNext, cancellationToken); + return await ReloadAfterDelay(fromPositionInclusive, maxCount, prefetchJsonData, cancellationToken); } // check for gap in messages collection @@ -196,14 +197,14 @@ protected virtual async Task HandleGap(TReadAllPage page, long fro { if(page.Messages[i].Position + 1 != page.Messages[i + 1].Position) { - return await ReloadAfterDelay(fromPositionInclusive, maxCount, prefetchJsonData, readNext, cancellationToken); + return await ReloadAfterDelay(fromPositionInclusive, maxCount, prefetchJsonData, cancellationToken); } } return page; } - public async Task ReadAllBackwards( + public async ITask ReadAllBackwards( long fromPositionInclusive, int maxCount, bool prefetchJsonData, @@ -220,9 +221,9 @@ public async Task ReadAllBackwards( fromPositionInclusive, maxCount); - ReadNextAllPage readNext = (nextPosition, ct) => ReadAllBackwards(nextPosition, maxCount, prefetchJsonData, ct); - var page = await ReadAllBackwardsInternal(fromPositionInclusive, maxCount, prefetchJsonData, readNext, cancellationToken); - return await FilterExpired(page, readNext, cancellationToken); + //ReadNextAllPage readNext = (nextPosition, ct) => ReadAllBackwards(nextPosition, maxCount, prefetchJsonData, ct); + var page = await ReadAllBackwardsInternal(fromPositionInclusive, maxCount, prefetchJsonData, cancellationToken); + return await FilterExpired(page, cancellationToken); } public async Task ReadStreamForwards( @@ -402,7 +403,6 @@ protected abstract Task ReadAllForwardsInternal( long fromPositionExclusive, int maxCount, bool prefetch, - ReadNextAllPage readNext, CancellationToken cancellationToken, long fromMaxPositionInclusive = -1); @@ -410,7 +410,6 @@ protected abstract Task ReadAllBackwardsInternal( long fromPositionExclusive, int maxCount, bool prefetch, - ReadNextAllPage readNext, CancellationToken cancellationToken); protected abstract Task ReadStreamForwardsInternal( @@ -477,14 +476,13 @@ private async Task ReloadAfterDelay( long fromPositionInclusive, int maxCount, bool prefetch, - ReadNextAllPage readNext, CancellationToken cancellationToken) { Logger.Info("ReadAllForwards: gap detected in position, reloading after {DefaultReloadInterval}ms, position: {fromPositionInclusive}", DefaultReloadInterval, fromPositionInclusive); //await Task.Delay(DefaultReloadInterval, cancellationToken); - var reloadedPage = await ReadAllForwardsInternal(fromPositionInclusive, maxCount, prefetch, readNext, cancellationToken) + var reloadedPage = await ReadAllForwardsInternal(fromPositionInclusive, maxCount, prefetch, cancellationToken) .ConfigureAwait(false); - return await FilterExpired(reloadedPage, readNext, cancellationToken).ConfigureAwait(false); + return await FilterExpired(reloadedPage, cancellationToken).ConfigureAwait(false); } private async Task FilterExpired( @@ -531,7 +529,6 @@ private async Task FilterExpired( } private async Task FilterExpired(TReadAllPage readAllPage, - ReadNextAllPage readNext, CancellationToken cancellationToken) { return readAllPage; diff --git a/src/SqlStreamStore/Infrastructure/StreamStoreBase.cs b/src/SqlStreamStore/Infrastructure/StreamStoreBase.cs index 870a246a4..782505062 100644 --- a/src/SqlStreamStore/Infrastructure/StreamStoreBase.cs +++ b/src/SqlStreamStore/Infrastructure/StreamStoreBase.cs @@ -12,7 +12,7 @@ namespace SqlStreamStore.Infrastructure /// Represents a base implementation of a stream store. It's primary purpose is to handle /// common needs across all derived implementations such as guard clauses and logging. /// - public abstract class StreamStoreBase : ReadonlyStreamStoreBase, IStreamStore where TReadAllPage : IReadAllPage + public abstract class StreamStoreBase : ReadonlyStreamStoreBase, IStreamStore where TReadAllPage : IReadAllPage { private readonly TaskQueue _taskQueue = new TaskQueue(); diff --git a/src/SqlStreamStore/SqlStreamStore.csproj b/src/SqlStreamStore/SqlStreamStore.csproj index 56b7e7087..da4db52e8 100644 --- a/src/SqlStreamStore/SqlStreamStore.csproj +++ b/src/SqlStreamStore/SqlStreamStore.csproj @@ -2,7 +2,7 @@ A library for writing and reading streams (also known as an 'event store') with a focus on SQL based implementations. This contains the core interfaces, abstractions and an in-memory implementation for tests. SQL Stream Store - netstandard2.0 + net5.0 $(DefineConstants);LIBLOG_PUBLIC true SqlStreamStore @@ -14,6 +14,7 @@ + all runtime; build; native; contentfiles; analyzers diff --git a/src/SqlStreamStore/StreamStoreExtensions.cs b/src/SqlStreamStore/StreamStoreExtensions.cs index 263cac63d..fbbf93167 100644 --- a/src/SqlStreamStore/StreamStoreExtensions.cs +++ b/src/SqlStreamStore/StreamStoreExtensions.cs @@ -2,6 +2,7 @@ { using System.Threading; using System.Threading.Tasks; + using MorseCode.ITask; using SqlStreamStore.Streams; /// @@ -29,7 +30,7 @@ public static class StreamStoreExtensions /// An presenting the result of the read. If all messages read have expired /// then the message collection MAY be empty. /// - public static Task ReadAllForwards( + public static ITask ReadAllForwards( this IReadonlyStreamStore readonlyStreamStore, long fromPositionInclusive, int maxCount, @@ -61,7 +62,7 @@ public static Task ReadAllForwards( /// An presenting the result of the read. If all messages read have expired /// then the message collection MAY be empty. /// - public static Task ReadAllBackwards( + public static ITask ReadAllBackwards( this IReadonlyStreamStore readonlyStreamStore, long fromPositionInclusive, int maxCount, @@ -101,7 +102,7 @@ public static Task ReadStreamForwards( string streamId, int fromVersionInclusive, int maxCount, - CancellationToken cancellationToken = default) where TReadAllPage : ReadAllPage, new() + CancellationToken cancellationToken = default) where TReadAllPage : IReadAllPage { return readonlyStreamStore.ReadStreamForwards( streamId, diff --git a/src/SqlStreamStore/Streams/ReadAllPage.cs b/src/SqlStreamStore/Streams/ReadAllPage.cs index 2e1d51c30..f8557e284 100644 --- a/src/SqlStreamStore/Streams/ReadAllPage.cs +++ b/src/SqlStreamStore/Streams/ReadAllPage.cs @@ -1,9 +1,5 @@ namespace SqlStreamStore.Streams { - using System.Threading; - using System.Threading.Tasks; - - public interface IReadAllPage { long FromPosition { get; } @@ -15,7 +11,7 @@ public interface IReadAllPage public class ReadAllPage : ReadAllPage { - public ReadAllPage(long fromPosition, long nextPosition, bool isEnd, ReadDirection direction, ReadNextAllPage readNext, StreamMessage[] messages = null) : base(fromPosition, nextPosition, isEnd, direction, readNext, messages) + public ReadAllPage(long fromPosition, long nextPosition, bool isEnd, ReadDirection direction, StreamMessage[] messages = null) : base(fromPosition, nextPosition, isEnd, direction, messages) { } } @@ -24,7 +20,7 @@ public ReadAllPage(long fromPosition, long nextPosition, bool isEnd, ReadDirecti /// public abstract class ReadAllPage : IReadAllPage where TReadAllPage : IReadAllPage { - private readonly ReadNextAllPage _readNext; + //private readonly ReadNextAllPage _readNext; /// /// A long representing the position where this page was read from. @@ -58,21 +54,18 @@ public abstract class ReadAllPage : IReadAllPage where TReadAllPag /// A long representing the position where the next page should be read from. /// True if page reach end of the all stream at time of reading. Otherwise false. /// The direction of the the read request. - /// An operation to read the next page of messages. /// The collection messages read. protected ReadAllPage( long fromPosition, long nextPosition, bool isEnd, ReadDirection direction, - ReadNextAllPage readNext, StreamMessage[] messages = null) { FromPosition = fromPosition; NextPosition = nextPosition; IsEnd = isEnd; Direction = direction; - _readNext = readNext; Messages = messages ?? new StreamMessage[0]; } @@ -83,14 +76,14 @@ public override string ToString() $"IsEnd: {IsEnd}, Direction: {Direction}, SteamEventCount: {Messages.Length}"; } - /// - /// Reads the next page. - /// - /// A token to cancel the operations. - /// A task the represents the asyncronous operation. - public Task ReadNext(CancellationToken cancellationToken = default) - { - return _readNext(NextPosition, cancellationToken); - } + ///// + ///// Reads the next page. + ///// + ///// A token to cancel the operations. + ///// A task the represents the asyncronous operation. + //public Task ReadNext(CancellationToken cancellationToken = default) + //{ + // return _readNext(NextPosition, cancellationToken); + //} } } \ No newline at end of file diff --git a/src/SqlStreamStore/Streams/ReadNextAllPage.cs b/src/SqlStreamStore/Streams/ReadNextAllPage.cs index 77d945c16..9b4d676a3 100644 --- a/src/SqlStreamStore/Streams/ReadNextAllPage.cs +++ b/src/SqlStreamStore/Streams/ReadNextAllPage.cs @@ -3,6 +3,9 @@ using System.Threading; using System.Threading.Tasks; + //using System.Threading; + //using System.Threading.Tasks; + /// /// Represents an operation to read the next all page. /// diff --git a/tests/Directory.Build.props b/tests/Directory.Build.props index 1c0923a44..bcbfc25fc 100644 --- a/tests/Directory.Build.props +++ b/tests/Directory.Build.props @@ -2,7 +2,7 @@ $(NoWarn);1701;1702;1705;1591;CS0618 - 7.3 + 9.0 true diff --git a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ReadAll.cs b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ReadAll.cs index ef068c035..16b6b680f 100644 --- a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ReadAll.cs +++ b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ReadAll.cs @@ -31,7 +31,7 @@ public async Task Can_read_all_forwards() int count = 0; while(!page.IsEnd && count <20) //should not take more than 20 iterations. { - page = await page.ReadNext(); + page = await Store.ReadAllForwards(page.NextPosition, 500); messages.AddRange(page.Messages); count++; } diff --git a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.cs b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.cs index ed550e769..ee64f4ad7 100644 --- a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.cs +++ b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.cs @@ -27,7 +27,7 @@ public async Task InitializeAsync() Fixture = await CreateFixture(); } - private IStreamStore Store => Fixture.Store; + private IStreamStore Store => Fixture.Store; protected IStreamStoreFixture Fixture { get; private set; } diff --git a/tests/SqlStreamStore.AcceptanceTests/IStreamStoreFixture.cs b/tests/SqlStreamStore.AcceptanceTests/IStreamStoreFixture.cs index af43de944..8b127d197 100644 --- a/tests/SqlStreamStore.AcceptanceTests/IStreamStoreFixture.cs +++ b/tests/SqlStreamStore.AcceptanceTests/IStreamStoreFixture.cs @@ -5,7 +5,7 @@ namespace SqlStreamStore public interface IStreamStoreFixture: IDisposable { - IStreamStore Store { get; } + IStreamStore Store { get; } GetUtcNow GetUtcNow { get; set; } diff --git a/tests/SqlStreamStore.Http.Tests/SqlStreamStore.Http.Tests.csproj b/tests/SqlStreamStore.Http.Tests/SqlStreamStore.Http.Tests.csproj index 6ebc9faf6..a7bba7dd7 100644 --- a/tests/SqlStreamStore.Http.Tests/SqlStreamStore.Http.Tests.csproj +++ b/tests/SqlStreamStore.Http.Tests/SqlStreamStore.Http.Tests.csproj @@ -1,7 +1,7 @@  - netcoreapp3.1 + net5.0 SqlStreamStore.Http.Tests SqlStreamStore.Http.Tests SqlStreamStore diff --git a/tests/SqlStreamStore.MsSql.Tests/SqlStreamStore.MsSql.Tests.csproj b/tests/SqlStreamStore.MsSql.Tests/SqlStreamStore.MsSql.Tests.csproj index a7a500e40..09be48eb0 100644 --- a/tests/SqlStreamStore.MsSql.Tests/SqlStreamStore.MsSql.Tests.csproj +++ b/tests/SqlStreamStore.MsSql.Tests/SqlStreamStore.MsSql.Tests.csproj @@ -1,6 +1,6 @@  - netcoreapp3.1 + net5.0 SqlStreamStore.MsSql.Tests SqlStreamStore.MsSql.Tests SqlStreamStore diff --git a/tests/SqlStreamStore.MsSql.V3.Tests/SqlStreamStore.MsSql.V3.Tests.csproj b/tests/SqlStreamStore.MsSql.V3.Tests/SqlStreamStore.MsSql.V3.Tests.csproj index 419e38c6e..69ab51192 100644 --- a/tests/SqlStreamStore.MsSql.V3.Tests/SqlStreamStore.MsSql.V3.Tests.csproj +++ b/tests/SqlStreamStore.MsSql.V3.Tests/SqlStreamStore.MsSql.V3.Tests.csproj @@ -1,6 +1,6 @@  - netcoreapp3.1 + net5.0 SqlStreamStore.MsSql.V3.Tests SqlStreamStore diff --git a/tests/SqlStreamStore.MySql.Tests/SqlStreamStore.MySql.Tests.csproj b/tests/SqlStreamStore.MySql.Tests/SqlStreamStore.MySql.Tests.csproj index 00a35b777..6911be4fb 100644 --- a/tests/SqlStreamStore.MySql.Tests/SqlStreamStore.MySql.Tests.csproj +++ b/tests/SqlStreamStore.MySql.Tests/SqlStreamStore.MySql.Tests.csproj @@ -1,6 +1,6 @@  - netcoreapp3.1 + net5.0 SqlStreamStore.MySql.Tests SqlStreamStore diff --git a/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreFixture.cs b/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreFixture.cs index 782de3a2e..3b7027c4c 100644 --- a/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreFixture.cs +++ b/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreFixture.cs @@ -41,7 +41,7 @@ public void Dispose() public string DatabaseName { get; } - public IStreamStore Store => PostgresStreamStore; + public IStreamStore Store => PostgresStreamStore; public PostgresStreamStore PostgresStreamStore { get; private set; } diff --git a/tests/SqlStreamStore.Postgres.Tests/SqlStreamStore.Postgres.Tests.csproj b/tests/SqlStreamStore.Postgres.Tests/SqlStreamStore.Postgres.Tests.csproj index 23e02cc30..f5a6f27eb 100644 --- a/tests/SqlStreamStore.Postgres.Tests/SqlStreamStore.Postgres.Tests.csproj +++ b/tests/SqlStreamStore.Postgres.Tests/SqlStreamStore.Postgres.Tests.csproj @@ -1,7 +1,7 @@  - netcoreapp3.1 + net5.0 SqlStreamStore.Postgres.Tests SqlStreamStore diff --git a/tests/SqlStreamStore.Sqlite.Tests/SqlStreamStore.Sqlite.Tests.csproj b/tests/SqlStreamStore.Sqlite.Tests/SqlStreamStore.Sqlite.Tests.csproj index 6c5ebb697..6dbdbbb07 100644 --- a/tests/SqlStreamStore.Sqlite.Tests/SqlStreamStore.Sqlite.Tests.csproj +++ b/tests/SqlStreamStore.Sqlite.Tests/SqlStreamStore.Sqlite.Tests.csproj @@ -1,6 +1,6 @@ - netcoreapp3.1 + net5.0 SqlStreamStore.Sqlite.Tests SqlStreamStore From dd5aca420a2cb92a7d6d6251b94f09ebbac142c5 Mon Sep 17 00:00:00 2001 From: Arne Schoonvliet Date: Thu, 3 Feb 2022 07:02:03 +0100 Subject: [PATCH 05/13] Version bumps --- src/LoadTests/LoadTests.csproj | 3 ++- src/LoadTests/Program.cs | 3 ++- .../PostgresStreamStore.cs | 16 +++++++------- .../Postgres/LibLogNpgsqlLogProvider.cs | 2 +- .../Postgres/PostgresContainer.cs | 3 ++- .../Postgres/PostgresDatabaseManager.cs | 4 ++-- .../SqlStreamStore.TestUtils.csproj | 22 +++++++++++-------- tests/Directory.Build.props | 4 ++-- 8 files changed, 32 insertions(+), 25 deletions(-) diff --git a/src/LoadTests/LoadTests.csproj b/src/LoadTests/LoadTests.csproj index d84344e19..7f7e4b8f1 100644 --- a/src/LoadTests/LoadTests.csproj +++ b/src/LoadTests/LoadTests.csproj @@ -8,8 +8,9 @@ - + + diff --git a/src/LoadTests/Program.cs b/src/LoadTests/Program.cs index 4d5f0ea8e..1ed271d94 100644 --- a/src/LoadTests/Program.cs +++ b/src/LoadTests/Program.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using EasyConsole; using Serilog; + using Serilog.Events; internal class Program { @@ -13,7 +14,7 @@ static async Task Main(string[] args) { Log.Logger = new LoggerConfiguration() .WriteTo - .File("LoadTests.txt") + .Console(LogEventLevel.Information) .CreateLogger(); var cts = new CancellationTokenSource(); diff --git a/src/SqlStreamStore.Postgres/PostgresStreamStore.cs b/src/SqlStreamStore.Postgres/PostgresStreamStore.cs index 30964d34b..e4c193ca0 100644 --- a/src/SqlStreamStore.Postgres/PostgresStreamStore.cs +++ b/src/SqlStreamStore.Postgres/PostgresStreamStore.cs @@ -273,14 +273,14 @@ public string GetSchemaCreationScript() // // TODO: FIXIT // // Check for gap between last page and this. - // if (page.Messages[0].Position != fromPositionInclusive) - // { - // Logger.DebugFormat("Gap detected at lower page boundary. Potentially could have lost {lostMessageCount} events if the gap is transient", page.Messages[0].Position - fromPositionInclusive); - // page = await HandleGap(page, fromPositionInclusive, maxCount, prefetchJsonData, cancellationToken); - // //if (!page.IsEnd || page.Messages.Length == 1) - // // Logger.DebugFormat("Gap detected at lower page boundary. Potentially could have lost {lostMessageCount} events if the gap is transient", page.Messages[0].Position - fromPositionInclusive); - // //page = await ReloadAfterDelay(fromPositionInclusive, maxCount, prefetchJsonData, ReadNext, cancellationToken); - // } + // //if (page.Messages[0].Position != fromPositionInclusive) + // //{ + // // Logger.InfoFormat("Gap detected at lower page boundary. Potentially could have lost {lostMessageCount} events if the gap is transient", page.Messages[0].Position - fromPositionInclusive); + // // page = await HandleGap(page, fromPositionInclusive, maxCount, prefetchJsonData, cancellationToken); + // // //if (!page.IsEnd || page.Messages.Length == 1) + // // // Logger.DebugFormat("Gap detected at lower page boundary. Potentially could have lost {lostMessageCount} events if the gap is transient", page.Messages[0].Position - fromPositionInclusive); + // // //page = await ReloadAfterDelay(fromPositionInclusive, maxCount, prefetchJsonData, ReadNext, cancellationToken); + // //} // // check for gap in messages collection // for (int i = 0; i < page.Messages.Length - 1; i++) diff --git a/src/SqlStreamStore.TestUtils/Postgres/LibLogNpgsqlLogProvider.cs b/src/SqlStreamStore.TestUtils/Postgres/LibLogNpgsqlLogProvider.cs index d0d6116d2..32311e2b0 100644 --- a/src/SqlStreamStore.TestUtils/Postgres/LibLogNpgsqlLogProvider.cs +++ b/src/SqlStreamStore.TestUtils/Postgres/LibLogNpgsqlLogProvider.cs @@ -23,7 +23,7 @@ public LibLogNpgsqlLogger(ILog logger, string name) _name = name; } - public override bool IsEnabled(NpgsqlLogLevel level) => true; + public override bool IsEnabled(NpgsqlLogLevel level) => false; public override void Log(NpgsqlLogLevel level, int connectorId, string msg, Exception exception = null) => _logger.Info($@"[{level:G}] [{_name}] (Connector Id: {connectorId}); {msg}; {FormatOptionalException(exception)}"); diff --git a/src/SqlStreamStore.TestUtils/Postgres/PostgresContainer.cs b/src/SqlStreamStore.TestUtils/Postgres/PostgresContainer.cs index bc7969324..02554f23c 100644 --- a/src/SqlStreamStore.TestUtils/Postgres/PostgresContainer.cs +++ b/src/SqlStreamStore.TestUtils/Postgres/PostgresContainer.cs @@ -20,6 +20,7 @@ public class PostgresContainer : PostgresDatabaseManager public PostgresContainer(string databaseName) : base(databaseName) { + _containerService = new Builder() .UseContainer() .WithName(ContainerName) @@ -58,7 +59,7 @@ await Policy Host = "localhost", Pooling = true, MaxPoolSize = 1024, - IncludeErrorDetails = true + IncludeErrorDetails = false }; } } diff --git a/src/SqlStreamStore.TestUtils/Postgres/PostgresDatabaseManager.cs b/src/SqlStreamStore.TestUtils/Postgres/PostgresDatabaseManager.cs index 220246293..d2f739575 100644 --- a/src/SqlStreamStore.TestUtils/Postgres/PostgresDatabaseManager.cs +++ b/src/SqlStreamStore.TestUtils/Postgres/PostgresDatabaseManager.cs @@ -21,8 +21,8 @@ public abstract class PostgresDatabaseManager static PostgresDatabaseManager() { #if DEBUG - NpgsqlLogManager.IsParameterLoggingEnabled = true; - NpgsqlLogManager.Provider = new LibLogNpgsqlLogProvider(); + NpgsqlLogManager.IsParameterLoggingEnabled = false; + //NpgsqlLogManager.Provider = new LibLogNpgsqlLogProvider(); #endif } diff --git a/src/SqlStreamStore.TestUtils/SqlStreamStore.TestUtils.csproj b/src/SqlStreamStore.TestUtils/SqlStreamStore.TestUtils.csproj index 96f2ad28c..4207e9e58 100644 --- a/src/SqlStreamStore.TestUtils/SqlStreamStore.TestUtils.csproj +++ b/src/SqlStreamStore.TestUtils/SqlStreamStore.TestUtils.csproj @@ -5,17 +5,17 @@ false - - - - - + + + + + - - + + - - + + all @@ -25,4 +25,8 @@ + + + + \ No newline at end of file diff --git a/tests/Directory.Build.props b/tests/Directory.Build.props index bcbfc25fc..3e2cf52fc 100644 --- a/tests/Directory.Build.props +++ b/tests/Directory.Build.props @@ -7,8 +7,8 @@ - - + + all From 2b278cc8d2339326ce4ede3f4d33ab52e4624a4b Mon Sep 17 00:00:00 2001 From: Arne Schoonvliet Date: Tue, 15 Feb 2022 08:33:41 +0100 Subject: [PATCH 06/13] Upgrade to .net 6 and all nuget dependencies to latest version (#1) * WIP * Fix tests * Build improvements * Fix build * Make build work * Use correct image * Fix pgsql v2 build * Try again * Fix build * Use stable .net 6 sdk * diagnostic logging * Set RID to make test work * WIP versioning * Fix build * Add migration from schema v1 to schema v2 * Make build work * WORK * WORK * FIXIT * Does this work? * WIP * Fix build * Make build work * Use other image for ci and pr build * Test build with updated csproj * Remove unused reference Co-authored-by: Arne Schoonvliet --- .github/workflows/ci.yaml | 4 +- .github/workflows/pr.yml | 4 +- .vscode/launch.json | 2 +- Dockerfile | 17 +- SqlStreamStore.sln | 16 +- build-local.cmd | 2 +- build.sh | 2 +- build/Program.cs | 16 +- build/build.csproj | 6 +- src/Directory.Build.props | 6 +- src/LoadTests/LoadTests.csproj | 11 +- .../SqlStreamStore.HAL.DevServer.csproj | 27 +- src/SqlStreamStore.HAL/Links.cs | 5 - src/SqlStreamStore.HAL/LoggingMiddleware.cs | 7 +- .../SqlStreamStore.HAL.csproj | 15 +- .../SqlStreamStoreHalMiddleware.cs | 5 - .../SqlStreamStore.Http.csproj | 8 +- .../SqlStreamStore.MsSql.csproj | 2 +- .../SqlStreamStore.MySql.csproj | 2 +- .../AppendToStream.sql | 0 .../DeleteStream.sql | 0 .../DeleteStreamMessages.sql | 0 .../DropAll.sql | 0 .../EnableExplainAnalyze.sql | 0 .../EnforceIdempotentAppend.sql | 0 .../ListStreams.sql | 0 .../ListStreamsEndingWith.sql | 0 .../ListStreamsStartingWith.sql | 0 .../Parameters.cs | 2 +- .../{PgSqlScripts => PgSqlScriptsV1}/Read.sql | 0 .../ReadAll.sql | 0 .../ReadHeadPosition.sql | 0 .../ReadJsonData.sql | 0 .../ReadSchemaVersion.sql | 0 .../ReadStreamHeadPosition.sql | 0 .../ReadStreamHeadVersion.sql | 0 .../ReadStreamVersionOfMessageId.sql | 0 .../Scavenge.sql | 0 .../Schema.cs | 2 +- .../Scripts.cs | 2 +- .../SetStreamMetadata.sql | 0 .../Tables.sql | 0 .../PostgresStreamStore.Append.cs | 2 +- .../PostgresStreamStore.Delete.cs | 2 +- .../PostgresStreamStore.ListStreams.cs | 2 +- .../PostgresStreamStore.Metadata.cs | 2 +- .../PostgresStreamStore.Read.cs | 2 +- .../PostgresStreamStore.ReadAll.cs | 2 +- .../PostgresStreamStore.cs | 3 +- .../PostgresStreamStoreSettings.cs | 1 + .../Properties/AssemblyInfo.cs | 3 +- .../SqlStreamStore.Postgres.csproj | 5 +- .../PgSqlScriptsV2/AppendToStream.sql | 177 +++++++++++ .../PgSqlScriptsV2/DeleteStream.sql | 73 +++++ .../PgSqlScriptsV2/DeleteStreamMessages.sql | 46 +++ .../PgSqlScriptsV2/DropAll.sql | 78 +++++ .../PgSqlScriptsV2/EnableExplainAnalyze.sql | 7 + .../EnforceIdempotentAppend.sql | 38 +++ .../PgSqlScriptsV2/ListStreams.sql | 19 ++ .../PgSqlScriptsV2/ListStreamsEndingWith.sql | 21 ++ .../ListStreamsStartingWith.sql | 21 ++ .../PgSqlScriptsV2/Migration.sql | 3 + .../PgSqlScriptsV2/Parameters.cs | 272 +++++++++++++++++ .../PgSqlScriptsV2/Read.sql | 53 ++++ .../PgSqlScriptsV2/ReadAll.sql | 47 +++ .../PgSqlScriptsV2/ReadHeadPosition.sql | 8 + .../PgSqlScriptsV2/ReadJsonData.sql | 16 + .../PgSqlScriptsV2/ReadSchemaVersion.sql | 13 + .../PgSqlScriptsV2/ReadStreamHeadPosition.sql | 10 + .../PgSqlScriptsV2/ReadStreamHeadVersion.sql | 10 + .../ReadStreamVersionOfMessageId.sql | 14 + .../PgSqlScriptsV2/Scavenge.sql | 33 +++ .../PgSqlScriptsV2/Schema.cs | 40 +++ .../PgSqlScriptsV2/Scripts.cs | 100 +++++++ .../PgSqlScriptsV2/SetStreamMetadata.sql | 38 +++ .../PgSqlScriptsV2/Tables.sql | 62 ++++ .../PostgresStreamStoreV2.Append.cs | 91 ++++++ .../PostgresStreamStoreV2.Delete.cs | 111 +++++++ .../PostgresStreamStoreV2.ListStreams.cs | 77 +++++ .../PostgresStreamStoreV2.Metadata.cs | 101 +++++++ .../PostgresStreamStoreV2.Read.cs | 278 ++++++++++++++++++ .../PostgresStreamStoreV2.ReadAll.cs | 212 +++++++++++++ .../PostgresStreamStoreV2.Subscriptions.cs | 47 +++ .../PostgresStreamStoreV2.cs | 272 +++++++++++++++++ .../PostgresStreamStoreV2Settings.cs | 100 +++++++ .../Properties/AssemblyInfo.cs | 3 + .../SqlStreamStore.PostgresV2.csproj | 20 ++ ...treamStore.SchemaCreationScriptTool.csproj | 10 +- .../SqlStreamStore.Sqlite.csproj | 4 +- .../MsSql/SqlServerContainer.cs | 2 +- .../SqlStreamStore.TestUtils.csproj | 22 +- src/SqlStreamStore/SqlStreamStore.csproj | 4 +- tests/Directory.Build.props | 4 +- .../AcceptanceTests.StreamLimits.cs | 8 +- .../SqlStreamStore.HAL.Tests.csproj | 4 +- .../SqlStreamStore.Http.Tests.csproj | 4 +- .../SqlStreamStore.MsSql.Tests.csproj | 3 +- .../SqlStreamStore.MsSql.V3.Tests.csproj | 3 +- .../SqlStreamStore.MySql.Tests.csproj | 3 +- .../SqlStreamStore.Postgres.Tests.csproj | 2 +- .../PostgresStreamStoreAcceptanceTests.cs | 157 ++++++++++ .../PostgresStreamStoreFixture.cs | 103 +++++++ .../PostgresStreamStoreFixturePool.cs | 58 ++++ .../SqlStreamStore.Postgres.V2.Tests.csproj | 16 + .../SqlStreamStore.Sqlite.Tests.csproj | 5 +- .../SqlStreamStore.Tests.csproj | 2 +- tools/hal-docs/gen-hal-docs.ps1 | 2 +- 107 files changed, 2991 insertions(+), 123 deletions(-) rename src/SqlStreamStore.Postgres/{PgSqlScripts => PgSqlScriptsV1}/AppendToStream.sql (100%) rename src/SqlStreamStore.Postgres/{PgSqlScripts => PgSqlScriptsV1}/DeleteStream.sql (100%) rename src/SqlStreamStore.Postgres/{PgSqlScripts => PgSqlScriptsV1}/DeleteStreamMessages.sql (100%) rename src/SqlStreamStore.Postgres/{PgSqlScripts => PgSqlScriptsV1}/DropAll.sql (100%) rename src/SqlStreamStore.Postgres/{PgSqlScripts => PgSqlScriptsV1}/EnableExplainAnalyze.sql (100%) rename src/SqlStreamStore.Postgres/{PgSqlScripts => PgSqlScriptsV1}/EnforceIdempotentAppend.sql (100%) rename src/SqlStreamStore.Postgres/{PgSqlScripts => PgSqlScriptsV1}/ListStreams.sql (100%) rename src/SqlStreamStore.Postgres/{PgSqlScripts => PgSqlScriptsV1}/ListStreamsEndingWith.sql (100%) rename src/SqlStreamStore.Postgres/{PgSqlScripts => PgSqlScriptsV1}/ListStreamsStartingWith.sql (100%) rename src/SqlStreamStore.Postgres/{PgSqlScripts => PgSqlScriptsV1}/Parameters.cs (99%) rename src/SqlStreamStore.Postgres/{PgSqlScripts => PgSqlScriptsV1}/Read.sql (100%) rename src/SqlStreamStore.Postgres/{PgSqlScripts => PgSqlScriptsV1}/ReadAll.sql (100%) rename src/SqlStreamStore.Postgres/{PgSqlScripts => PgSqlScriptsV1}/ReadHeadPosition.sql (100%) rename src/SqlStreamStore.Postgres/{PgSqlScripts => PgSqlScriptsV1}/ReadJsonData.sql (100%) rename src/SqlStreamStore.Postgres/{PgSqlScripts => PgSqlScriptsV1}/ReadSchemaVersion.sql (100%) rename src/SqlStreamStore.Postgres/{PgSqlScripts => PgSqlScriptsV1}/ReadStreamHeadPosition.sql (100%) rename src/SqlStreamStore.Postgres/{PgSqlScripts => PgSqlScriptsV1}/ReadStreamHeadVersion.sql (100%) rename src/SqlStreamStore.Postgres/{PgSqlScripts => PgSqlScriptsV1}/ReadStreamVersionOfMessageId.sql (100%) rename src/SqlStreamStore.Postgres/{PgSqlScripts => PgSqlScriptsV1}/Scavenge.sql (100%) rename src/SqlStreamStore.Postgres/{PgSqlScripts => PgSqlScriptsV1}/Schema.cs (97%) rename src/SqlStreamStore.Postgres/{PgSqlScripts => PgSqlScriptsV1}/Scripts.cs (98%) rename src/SqlStreamStore.Postgres/{PgSqlScripts => PgSqlScriptsV1}/SetStreamMetadata.sql (100%) rename src/SqlStreamStore.Postgres/{PgSqlScripts => PgSqlScriptsV1}/Tables.sql (100%) create mode 100644 src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/AppendToStream.sql create mode 100644 src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/DeleteStream.sql create mode 100644 src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/DeleteStreamMessages.sql create mode 100644 src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/DropAll.sql create mode 100644 src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/EnableExplainAnalyze.sql create mode 100644 src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/EnforceIdempotentAppend.sql create mode 100644 src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ListStreams.sql create mode 100644 src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ListStreamsEndingWith.sql create mode 100644 src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ListStreamsStartingWith.sql create mode 100644 src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Migration.sql create mode 100644 src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Parameters.cs create mode 100644 src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Read.sql create mode 100644 src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadAll.sql create mode 100644 src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadHeadPosition.sql create mode 100644 src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadJsonData.sql create mode 100644 src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadSchemaVersion.sql create mode 100644 src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadStreamHeadPosition.sql create mode 100644 src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadStreamHeadVersion.sql create mode 100644 src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadStreamVersionOfMessageId.sql create mode 100644 src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Scavenge.sql create mode 100644 src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Schema.cs create mode 100644 src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Scripts.cs create mode 100644 src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/SetStreamMetadata.sql create mode 100644 src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Tables.sql create mode 100644 src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Append.cs create mode 100644 src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Delete.cs create mode 100644 src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.ListStreams.cs create mode 100644 src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Metadata.cs create mode 100644 src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Read.cs create mode 100644 src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.ReadAll.cs create mode 100644 src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Subscriptions.cs create mode 100644 src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.cs create mode 100644 src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2Settings.cs create mode 100644 src/SqlStreamStore.PostgresV2/Properties/AssemblyInfo.cs create mode 100644 src/SqlStreamStore.PostgresV2/SqlStreamStore.PostgresV2.csproj create mode 100644 tests/SqlStreamStore.Postgres.V2.Tests/PostgresStreamStoreAcceptanceTests.cs create mode 100644 tests/SqlStreamStore.Postgres.V2.Tests/PostgresStreamStoreFixture.cs create mode 100644 tests/SqlStreamStore.Postgres.V2.Tests/PostgresStreamStoreFixturePool.cs create mode 100644 tests/SqlStreamStore.Postgres.V2.Tests/SqlStreamStore.Postgres.V2.Tests.csproj diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 754d84b62..d7cc9001b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - test: ["inmem", "mssql", "mssql-v3", "mysql", "postgres", "sqlite", "hal", "http"] + test: ["inmem", "mssql", "mssql-v3", "mysql", "postgres", "postgres-v2", "sqlite", "hal", "http"] runs-on: ubuntu-latest name: Test ${{ matrix.test }} @@ -46,7 +46,7 @@ jobs: fetch-depth: 0 - name: Pull Containers shell: bash - run: docker pull damianh/dotnet-core-lts-sdks:1 + run: docker pull alpinelinux/docker-cli:latest-x86_64 - name: Build and Test shell: bash run: ./build.sh test-${{ matrix.test }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index bea46c839..56ac107a2 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -8,7 +8,7 @@ jobs: strategy: fail-fast: false matrix: - test: ["inmem", "mssql", "mssql-v3", "mysql", "postgres", "sqlite", "hal", "http"] + test: ["inmem", "mssql", "mssql-v3", "mysql", "postgres", "postgres-v2", "sqlite", "hal", "http"] runs-on: ubuntu-latest name: Test ${{ matrix.test }} @@ -41,7 +41,7 @@ jobs: fetch-depth: 0 - name: Pull Containers shell: bash - run: docker pull damianh/dotnet-core-lts-sdks:1 + run: docker pull alpinelinux/docker-cli:latest-x86_64 - name: Build and Test shell: bash run: ./build.sh test-${{ matrix.test }} diff --git a/.vscode/launch.json b/.vscode/launch.json index 22bd38eb9..9c9a807d9 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -10,7 +10,7 @@ "request": "launch", "preLaunchTask": "build", // If you have changed target frameworks, make sure to update the program path. - "program": "${workspaceFolder}/src/LoadTests/bin/Debug/netcoreapp2.0/LoadTests.dll", + "program": "${workspaceFolder}/src/LoadTests/bin/Debug/net6.0/LoadTests.dll", "args": [], "cwd": "${workspaceFolder}/src/LoadTests", // For more information about the 'console' field, see https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md#console-terminal-window diff --git a/Dockerfile b/Dockerfile index 7cd4334a7..1354b9bd4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,19 @@ -FROM damianh/dotnet-core-lts-sdks:2 +FROM mcr.microsoft.com/dotnet/sdk:6.0-focal + +RUN apt-get update \ + && apt-get install git \ + && apt-get install \ + ca-certificates \ + curl \ + gnupg \ + lsb-release -y \ + && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg \ + && echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null \ + && apt-get update \ + && apt-get install docker-ce-cli -y \ + && apt-get install zip -y WORKDIR /repo diff --git a/SqlStreamStore.sln b/SqlStreamStore.sln index 60ca1bd2f..94cef37f9 100644 --- a/SqlStreamStore.sln +++ b/SqlStreamStore.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.29926.136 +# Visual Studio Version 17 +VisualStudioVersion = 17.1.32120.378 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SqlStreamStore", "src\SqlStreamStore\SqlStreamStore.csproj", "{8F86833A-6686-409C-B0E4-6FBFB70F7DE1}" EndProject @@ -62,6 +62,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{845ED623-9F0 EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "build", "build\build.csproj", "{5273D70A-96D7-4775-B67C-BDE48D20CA36}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SqlStreamStore.Postgres.V2.Tests", "tests\SqlStreamStore.Postgres.V2.Tests\SqlStreamStore.Postgres.V2.Tests.csproj", "{A8D87EDC-E911-42F3-891F-82DEBBD4E0A3}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SqlStreamStore.PostgresV2", "src\SqlStreamStore.PostgresV2\SqlStreamStore.PostgresV2.csproj", "{D1FA4E2A-9113-4F8E-987D-DDC1C1C583E2}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -148,6 +152,14 @@ Global {5273D70A-96D7-4775-B67C-BDE48D20CA36}.Debug|Any CPU.Build.0 = Debug|Any CPU {5273D70A-96D7-4775-B67C-BDE48D20CA36}.Release|Any CPU.ActiveCfg = Release|Any CPU {5273D70A-96D7-4775-B67C-BDE48D20CA36}.Release|Any CPU.Build.0 = Release|Any CPU + {A8D87EDC-E911-42F3-891F-82DEBBD4E0A3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A8D87EDC-E911-42F3-891F-82DEBBD4E0A3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A8D87EDC-E911-42F3-891F-82DEBBD4E0A3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A8D87EDC-E911-42F3-891F-82DEBBD4E0A3}.Release|Any CPU.Build.0 = Release|Any CPU + {D1FA4E2A-9113-4F8E-987D-DDC1C1C583E2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D1FA4E2A-9113-4F8E-987D-DDC1C1C583E2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D1FA4E2A-9113-4F8E-987D-DDC1C1C583E2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D1FA4E2A-9113-4F8E-987D-DDC1C1C583E2}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/build-local.cmd b/build-local.cmd index ac72a1451..6eff8ed53 100644 --- a/build-local.cmd +++ b/build-local.cmd @@ -1,3 +1,3 @@ @ECHO OFF -dotnet run -p build/build.csproj -- %* \ No newline at end of file +dotnet run --project build/build.csproj -- %* \ No newline at end of file diff --git a/build.sh b/build.sh index 9b2d1f61e..c6fb9c9a9 100755 --- a/build.sh +++ b/build.sh @@ -8,4 +8,4 @@ docker run --rm --name sss-build \ --network host \ -e FEEDZ_SSS_API_KEY=$FEEDZ_SSS_API_KEY \ sss-build \ - dotnet run -p /repo/build/build.csproj -- "$@" \ No newline at end of file + dotnet run --project /repo/build/build.csproj -- "$@" \ No newline at end of file diff --git a/build/Program.cs b/build/Program.cs index e87410066..b230dbea8 100644 --- a/build/Program.cs +++ b/build/Program.cs @@ -8,6 +8,8 @@ namespace build { + using System.Text; + class Program { private const string ArtifactsDir = "artifacts"; @@ -20,6 +22,7 @@ class Program private const string TestMsSql = "test-mssql"; private const string TestMsSqlV3 = "test-mssql-v3"; private const string TestPostgres = "test-postgres"; + private const string TestPostgresV2 = "test-postgres-v2"; private const string TestSqlite = "test-sqlite"; private const string TestHal = "test-hal"; private const string TestHttp = "test-http"; @@ -59,11 +62,10 @@ void RunTest(string project) { try { - Run("dotnet", - $"test tests/{project}/{project}.csproj --configuration=Release --no-build --no-restore --verbosity=normal" - + $" --logger \"trx;logfilename=..\\..\\..\\{ArtifactsDir}\\{project}.trx\""); + Run("dotnet", $"test tests/{project}/{project}.csproj --configuration=Release --no-build --no-restore --verbosity=normal" + + $" --logger \"trx;logfilename=..\\..\\..\\{ArtifactsDir}\\{project}.trx\""); } - catch (NonZeroExitCodeException) when (ShouldCatch()) + catch (ExitCodeException) when (ShouldCatch()) { TestProjectsWithFailures.Add(project); } @@ -106,6 +108,11 @@ void RunTest(string project) DependsOn(Build), () => RunTest("SqlStreamStore.Postgres.Tests")); + Target( + TestPostgresV2, + DependsOn(Build), + () => RunTest("SqlStreamStore.Postgres.V2.Tests")); + Target( TestSqlite, DependsOn(Build), @@ -123,6 +130,7 @@ void RunTest(string project) "SqlStreamStore.MsSql", "SqlStreamStore.MySql", "SqlStreamStore.Postgres", + "SqlStreamStore.PostgresV2", "SqlStreamStore.HAL", "SqlStreamStore.Http", "SqlStreamStore.Sqlite", diff --git a/build/build.csproj b/build/build.csproj index 911b97451..b6b91514b 100644 --- a/build/build.csproj +++ b/build/build.csproj @@ -2,12 +2,12 @@ Exe - netcoreapp3.1 + net6.0 - - + + diff --git a/src/Directory.Build.props b/src/Directory.Build.props index c3a6e9c51..310b95e4e 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -9,14 +9,14 @@ $(NoWarn);1701;1702;1705;1591 7.3 v - 1.2 + 1.3 full true $(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb - - + + diff --git a/src/LoadTests/LoadTests.csproj b/src/LoadTests/LoadTests.csproj index b6a85c474..7e6cea046 100644 --- a/src/LoadTests/LoadTests.csproj +++ b/src/LoadTests/LoadTests.csproj @@ -1,18 +1,15 @@ - netcoreapp3.1 + net6.0 win10-x64;osx.10.11-x64;ubuntu.16.10-x64;ubuntu.14.04-x64 exe false - - - - - - + + + diff --git a/src/SqlStreamStore.HAL.DevServer/SqlStreamStore.HAL.DevServer.csproj b/src/SqlStreamStore.HAL.DevServer/SqlStreamStore.HAL.DevServer.csproj index 17c615302..d3925740a 100644 --- a/src/SqlStreamStore.HAL.DevServer/SqlStreamStore.HAL.DevServer.csproj +++ b/src/SqlStreamStore.HAL.DevServer/SqlStreamStore.HAL.DevServer.csproj @@ -1,26 +1,25 @@ - + - Exe - netcoreapp2.1 - true - false + net6.0 - - - - - - - - - + + + + + + + + + + + diff --git a/src/SqlStreamStore.HAL/Links.cs b/src/SqlStreamStore.HAL/Links.cs index 32ada51f0..82f6814d9 100644 --- a/src/SqlStreamStore.HAL/Links.cs +++ b/src/SqlStreamStore.HAL/Links.cs @@ -6,12 +6,9 @@ namespace SqlStreamStore.HAL using System.Net.Http; using Halcyon.HAL; using Microsoft.AspNetCore.Http; - using SqlStreamStore.HAL.Logging; internal class Links { - private static readonly ILog s_log = LogProvider.For(); - private readonly PathString _path; private readonly List<(string rel, string href, string title)> _links; private readonly string _relativePathToRoot; @@ -43,8 +40,6 @@ public Links Add(string rel, string href, string title = null) throw new ArgumentNullException(nameof(href)); _links.Add((rel, href, title)); - - s_log.Debug("Added link {link} to response for request {path}", _links[_links.Count - 1], _path); return this; } diff --git a/src/SqlStreamStore.HAL/LoggingMiddleware.cs b/src/SqlStreamStore.HAL/LoggingMiddleware.cs index dd03c8316..67684314f 100644 --- a/src/SqlStreamStore.HAL/LoggingMiddleware.cs +++ b/src/SqlStreamStore.HAL/LoggingMiddleware.cs @@ -4,17 +4,18 @@ namespace SqlStreamStore.HAL using System.Threading.Tasks; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Http; - using SqlStreamStore.HAL.Logging; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Logging; internal static class LoggingMiddleware { public static IApplicationBuilder UseMiddlewareLogging(this IApplicationBuilder builder, Type middlewareType) { - var log = LogProvider.GetLogger(middlewareType); + var log = builder.ApplicationServices.GetService().CreateLogger(middlewareType); Task MiddlewareLogging(HttpContext context, Func next) { - log.Debug("Middleware: {middleware}; Connection Id: {connectionId}", middlewareType, context.Connection.Id); + log.LogDebug("Middleware: {middleware}; Connection Id: {connectionId}", middlewareType, context.Connection.Id); return next(); } diff --git a/src/SqlStreamStore.HAL/SqlStreamStore.HAL.csproj b/src/SqlStreamStore.HAL/SqlStreamStore.HAL.csproj index c45a33b19..28eca7e68 100644 --- a/src/SqlStreamStore.HAL/SqlStreamStore.HAL.csproj +++ b/src/SqlStreamStore.HAL/SqlStreamStore.HAL.csproj @@ -1,26 +1,31 @@  - netstandard2.0 - $(DefineConstants);LIBLOG_PORTABLE + net5.0;net6.0 HTTP Server for SQL Stream Store Stream Store - HAL Server true false + + + + + - - - + + %(RecursiveDir)%(Filename)%(Extension) + + diff --git a/src/SqlStreamStore.HAL/SqlStreamStoreHalMiddleware.cs b/src/SqlStreamStore.HAL/SqlStreamStoreHalMiddleware.cs index 8a5ecc4c0..ebb3a5869 100644 --- a/src/SqlStreamStore.HAL/SqlStreamStoreHalMiddleware.cs +++ b/src/SqlStreamStore.HAL/SqlStreamStoreHalMiddleware.cs @@ -12,7 +12,6 @@ using SqlStreamStore.HAL.AllStreamMessage; using SqlStreamStore.HAL.Docs; using SqlStreamStore.HAL.Index; - using SqlStreamStore.HAL.Logging; using SqlStreamStore.HAL.StreamBrowser; using SqlStreamStore.HAL.StreamMessage; using SqlStreamStore.HAL.StreamMetadata; @@ -25,8 +24,6 @@ public static class SqlStreamStoreHalMiddleware { - private static ILog s_Log = LogProvider.GetLogger(typeof(SqlStreamStoreHalMiddleware)); - private static MidFunc Rfc1738 = (context, next) => { if(context.Request.QueryString != QueryString.Empty) @@ -78,8 +75,6 @@ public static IApplicationBuilder UseSqlStreamStoreHal( streamMetadata, streamBrowser); - s_Log.Info(index.ToString); - return builder .UseExceptionHandling() .Use(Rfc1738) diff --git a/src/SqlStreamStore.Http/SqlStreamStore.Http.csproj b/src/SqlStreamStore.Http/SqlStreamStore.Http.csproj index 6b58d086f..2f338f369 100644 --- a/src/SqlStreamStore.Http/SqlStreamStore.Http.csproj +++ b/src/SqlStreamStore.Http/SqlStreamStore.Http.csproj @@ -9,10 +9,10 @@ SqlStreamStore - - - - + + + + diff --git a/src/SqlStreamStore.MsSql/SqlStreamStore.MsSql.csproj b/src/SqlStreamStore.MsSql/SqlStreamStore.MsSql.csproj index 3c7d47698..a0dcb577f 100644 --- a/src/SqlStreamStore.MsSql/SqlStreamStore.MsSql.csproj +++ b/src/SqlStreamStore.MsSql/SqlStreamStore.MsSql.csproj @@ -17,6 +17,6 @@ - + \ No newline at end of file diff --git a/src/SqlStreamStore.MySql/SqlStreamStore.MySql.csproj b/src/SqlStreamStore.MySql/SqlStreamStore.MySql.csproj index 4328836b4..01afcb004 100644 --- a/src/SqlStreamStore.MySql/SqlStreamStore.MySql.csproj +++ b/src/SqlStreamStore.MySql/SqlStreamStore.MySql.csproj @@ -12,7 +12,7 @@ - + diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/AppendToStream.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/AppendToStream.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/AppendToStream.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/AppendToStream.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/DeleteStream.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/DeleteStream.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/DeleteStream.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/DeleteStream.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/DeleteStreamMessages.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/DeleteStreamMessages.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/DeleteStreamMessages.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/DeleteStreamMessages.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/DropAll.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/DropAll.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/DropAll.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/DropAll.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/EnableExplainAnalyze.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/EnableExplainAnalyze.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/EnableExplainAnalyze.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/EnableExplainAnalyze.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/EnforceIdempotentAppend.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/EnforceIdempotentAppend.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/EnforceIdempotentAppend.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/EnforceIdempotentAppend.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/ListStreams.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/ListStreams.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/ListStreams.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/ListStreams.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/ListStreamsEndingWith.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/ListStreamsEndingWith.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/ListStreamsEndingWith.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/ListStreamsEndingWith.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/ListStreamsStartingWith.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/ListStreamsStartingWith.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/ListStreamsStartingWith.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/ListStreamsStartingWith.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/Parameters.cs b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/Parameters.cs similarity index 99% rename from src/SqlStreamStore.Postgres/PgSqlScripts/Parameters.cs rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/Parameters.cs index 5659a08e9..aa621cb8e 100644 --- a/src/SqlStreamStore.Postgres/PgSqlScripts/Parameters.cs +++ b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/Parameters.cs @@ -1,4 +1,4 @@ -namespace SqlStreamStore.PgSqlScripts +namespace SqlStreamStore.PgSqlScriptsV1 { using System; using Npgsql; diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/Read.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/Read.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/Read.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/Read.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/ReadAll.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/ReadAll.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/ReadAll.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/ReadAll.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/ReadHeadPosition.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/ReadHeadPosition.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/ReadHeadPosition.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/ReadHeadPosition.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/ReadJsonData.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/ReadJsonData.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/ReadJsonData.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/ReadJsonData.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/ReadSchemaVersion.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/ReadSchemaVersion.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/ReadSchemaVersion.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/ReadSchemaVersion.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/ReadStreamHeadPosition.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/ReadStreamHeadPosition.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/ReadStreamHeadPosition.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/ReadStreamHeadPosition.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/ReadStreamHeadVersion.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/ReadStreamHeadVersion.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/ReadStreamHeadVersion.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/ReadStreamHeadVersion.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/ReadStreamVersionOfMessageId.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/ReadStreamVersionOfMessageId.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/ReadStreamVersionOfMessageId.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/ReadStreamVersionOfMessageId.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/Scavenge.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/Scavenge.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/Scavenge.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/Scavenge.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/Schema.cs b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/Schema.cs similarity index 97% rename from src/SqlStreamStore.Postgres/PgSqlScripts/Schema.cs rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/Schema.cs index deb1e09ab..899b51ffb 100644 --- a/src/SqlStreamStore.Postgres/PgSqlScripts/Schema.cs +++ b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/Schema.cs @@ -1,4 +1,4 @@ -namespace SqlStreamStore.PgSqlScripts +namespace SqlStreamStore.PgSqlScriptsV1 { internal class Schema { diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/Scripts.cs b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/Scripts.cs similarity index 98% rename from src/SqlStreamStore.Postgres/PgSqlScripts/Scripts.cs rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/Scripts.cs index c134ae440..757bd8416 100644 --- a/src/SqlStreamStore.Postgres/PgSqlScripts/Scripts.cs +++ b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/Scripts.cs @@ -1,4 +1,4 @@ -namespace SqlStreamStore.PgSqlScripts +namespace SqlStreamStore.PgSqlScriptsV1 { using System; using System.Collections.Concurrent; diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/SetStreamMetadata.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/SetStreamMetadata.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/SetStreamMetadata.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/SetStreamMetadata.sql diff --git a/src/SqlStreamStore.Postgres/PgSqlScripts/Tables.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/Tables.sql similarity index 100% rename from src/SqlStreamStore.Postgres/PgSqlScripts/Tables.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV1/Tables.sql diff --git a/src/SqlStreamStore.Postgres/PostgresStreamStore.Append.cs b/src/SqlStreamStore.Postgres/PostgresStreamStore.Append.cs index b8cc7fe49..493fcc02f 100644 --- a/src/SqlStreamStore.Postgres/PostgresStreamStore.Append.cs +++ b/src/SqlStreamStore.Postgres/PostgresStreamStore.Append.cs @@ -5,7 +5,7 @@ using System.Threading; using System.Threading.Tasks; using Npgsql; - using SqlStreamStore.PgSqlScripts; + using SqlStreamStore.PgSqlScriptsV1; using SqlStreamStore.Streams; partial class PostgresStreamStore diff --git a/src/SqlStreamStore.Postgres/PostgresStreamStore.Delete.cs b/src/SqlStreamStore.Postgres/PostgresStreamStore.Delete.cs index 0f4026f0f..aeaa1b885 100644 --- a/src/SqlStreamStore.Postgres/PostgresStreamStore.Delete.cs +++ b/src/SqlStreamStore.Postgres/PostgresStreamStore.Delete.cs @@ -4,7 +4,7 @@ using System.Threading; using System.Threading.Tasks; using Npgsql; - using SqlStreamStore.PgSqlScripts; + using SqlStreamStore.PgSqlScriptsV1; using SqlStreamStore.Streams; public partial class PostgresStreamStore diff --git a/src/SqlStreamStore.Postgres/PostgresStreamStore.ListStreams.cs b/src/SqlStreamStore.Postgres/PostgresStreamStore.ListStreams.cs index 9febddd6d..3e0a538be 100644 --- a/src/SqlStreamStore.Postgres/PostgresStreamStore.ListStreams.cs +++ b/src/SqlStreamStore.Postgres/PostgresStreamStore.ListStreams.cs @@ -4,7 +4,7 @@ namespace SqlStreamStore using System.Threading; using System.Threading.Tasks; using Npgsql; - using SqlStreamStore.PgSqlScripts; + using SqlStreamStore.PgSqlScriptsV1; using SqlStreamStore.Streams; public partial class PostgresStreamStore diff --git a/src/SqlStreamStore.Postgres/PostgresStreamStore.Metadata.cs b/src/SqlStreamStore.Postgres/PostgresStreamStore.Metadata.cs index 1c276119f..250fda9aa 100644 --- a/src/SqlStreamStore.Postgres/PostgresStreamStore.Metadata.cs +++ b/src/SqlStreamStore.Postgres/PostgresStreamStore.Metadata.cs @@ -3,7 +3,7 @@ using System.Threading; using System.Threading.Tasks; using Npgsql; - using SqlStreamStore.PgSqlScripts; + using SqlStreamStore.PgSqlScriptsV1; using SqlStreamStore.Streams; partial class PostgresStreamStore diff --git a/src/SqlStreamStore.Postgres/PostgresStreamStore.Read.cs b/src/SqlStreamStore.Postgres/PostgresStreamStore.Read.cs index 47e175271..6055679d7 100644 --- a/src/SqlStreamStore.Postgres/PostgresStreamStore.Read.cs +++ b/src/SqlStreamStore.Postgres/PostgresStreamStore.Read.cs @@ -9,7 +9,7 @@ using System.Threading; using System.Threading.Tasks; using Npgsql; - using SqlStreamStore.PgSqlScripts; + using SqlStreamStore.PgSqlScriptsV1; using SqlStreamStore.Streams; partial class PostgresStreamStore diff --git a/src/SqlStreamStore.Postgres/PostgresStreamStore.ReadAll.cs b/src/SqlStreamStore.Postgres/PostgresStreamStore.ReadAll.cs index 1eeedf0b5..50e3ded80 100644 --- a/src/SqlStreamStore.Postgres/PostgresStreamStore.ReadAll.cs +++ b/src/SqlStreamStore.Postgres/PostgresStreamStore.ReadAll.cs @@ -6,7 +6,7 @@ using System.Data.Common; using System.Threading; using System.Threading.Tasks; - using SqlStreamStore.PgSqlScripts; + using SqlStreamStore.PgSqlScriptsV1; using SqlStreamStore.Streams; partial class PostgresStreamStore diff --git a/src/SqlStreamStore.Postgres/PostgresStreamStore.cs b/src/SqlStreamStore.Postgres/PostgresStreamStore.cs index 32da063ed..97cde1e44 100644 --- a/src/SqlStreamStore.Postgres/PostgresStreamStore.cs +++ b/src/SqlStreamStore.Postgres/PostgresStreamStore.cs @@ -8,12 +8,13 @@ using Npgsql; using SqlStreamStore.Infrastructure; using SqlStreamStore.Logging; - using SqlStreamStore.PgSqlScripts; + using SqlStreamStore.PgSqlScriptsV1; using SqlStreamStore.Subscriptions; /// /// Represents a PostgreSQL stream store implementation. /// + [Obsolete("Use PostgresStreamStoreV2 instead. Note: this will require a schema and data migration.", false)] public partial class PostgresStreamStore : StreamStoreBase { private readonly PostgresStreamStoreSettings _settings; diff --git a/src/SqlStreamStore.Postgres/PostgresStreamStoreSettings.cs b/src/SqlStreamStore.Postgres/PostgresStreamStoreSettings.cs index 796d93fdd..f468994b6 100644 --- a/src/SqlStreamStore.Postgres/PostgresStreamStoreSettings.cs +++ b/src/SqlStreamStore.Postgres/PostgresStreamStoreSettings.cs @@ -6,6 +6,7 @@ using SqlStreamStore.Infrastructure; using SqlStreamStore.Subscriptions; + [Obsolete("Use PostgresStreamStoreV2Settings instead. Note: this will require a schema and data migration.", false)] public class PostgresStreamStoreSettings { private string _schema = "public"; diff --git a/src/SqlStreamStore.Postgres/Properties/AssemblyInfo.cs b/src/SqlStreamStore.Postgres/Properties/AssemblyInfo.cs index 8ddfbb782..96229f8c5 100644 --- a/src/SqlStreamStore.Postgres/Properties/AssemblyInfo.cs +++ b/src/SqlStreamStore.Postgres/Properties/AssemblyInfo.cs @@ -1,3 +1,4 @@ using System.Runtime.CompilerServices; -[assembly: InternalsVisibleTo("SqlStreamStore.Postgres.Tests")] \ No newline at end of file +[assembly: InternalsVisibleTo("SqlStreamStore.Postgres.Tests")] +[assembly: InternalsVisibleTo("SqlStreamStore.PostgresV2")] diff --git a/src/SqlStreamStore.Postgres/SqlStreamStore.Postgres.csproj b/src/SqlStreamStore.Postgres/SqlStreamStore.Postgres.csproj index 9702d8626..7b2a10afc 100644 --- a/src/SqlStreamStore.Postgres/SqlStreamStore.Postgres.csproj +++ b/src/SqlStreamStore.Postgres/SqlStreamStore.Postgres.csproj @@ -6,13 +6,12 @@ SqlStreamStore.Postgres SqlStreamStore.Postgres SqlStreamStore - 1.6.0 - + - + diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/AppendToStream.sql b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/AppendToStream.sql new file mode 100644 index 000000000..745a390af --- /dev/null +++ b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/AppendToStream.sql @@ -0,0 +1,177 @@ +CREATE OR REPLACE FUNCTION __schema__.append_to_stream( + _stream_id CHAR(42), + _stream_id_original VARCHAR(1000), + _metadata_stream_id CHAR(42), + _expected_version INT, + _created_utc TIMESTAMP WITH TIME ZONE, + _new_stream_messages __schema__.new_stream_message []) + RETURNS TABLE( + current_version INT, + current_position BIGINT + ) AS $F$ +DECLARE + _current_version INT; + _current_position BIGINT; + _stream_id_internal INT; + _success INT; + _max_age INT; + _max_count INT; +BEGIN + IF _created_utc IS NULL + THEN + _created_utc = now() at time zone 'utc'; + END IF; + + IF _expected_version < 0 + THEN + SELECT __schema__.messages.json_data :: JSON->>'MaxAge', __schema__.messages.json_data :: JSON->>'MaxCount' + INTO _max_age, _max_count + FROM __schema__.messages + JOIN __schema__.streams ON __schema__.messages.stream_id_internal = __schema__.streams.id_internal + WHERE __schema__.streams.id = _metadata_stream_id + ORDER BY __schema__.messages.stream_version DESC + LIMIT 1; + + INSERT INTO __schema__.streams (id, id_original, max_age, max_count) + SELECT _stream_id, _stream_id_original, _max_age, _max_count + ON CONFLICT DO NOTHING; + GET DIAGNOSTICS _success = ROW_COUNT; + + END IF; + + IF _expected_version = -1 /* ExpectedVersion.Empty */ + THEN + + IF _success = 0 AND + cardinality(_new_stream_messages) > 0 AND + (SELECT __schema__.streams.version FROM __schema__.streams WHERE __schema__.streams.id_internal = _stream_id_internal) > 0 + THEN + + RAISE EXCEPTION 'WrongExpectedVersion'; + END IF; + ELSIF _expected_version = -3 /* ExpectedVersion.NoStream */ + THEN + IF _success = 0 AND cardinality(_new_stream_messages) > 0 + THEN + PERFORM __schema__.enforce_idempotent_append( + _stream_id, + 0, + false, + _new_stream_messages); + SELECT version, position, id_internal + INTO _current_version, _current_position, _stream_id_internal + FROM __schema__.streams + WHERE __schema__.streams.id = _stream_id; + + RETURN QUERY + SELECT _current_version, _current_position; + RETURN; + END IF; + END IF; + + SELECT (CASE _expected_version + WHEN -2 /* ExpectedVersion.Any */ + THEN coalesce( + __schema__.read_stream_version_of_message_id( + __schema__.streams.id_internal, + _new_stream_messages [ 1 ].message_id) - 1, + __schema__.streams.version) + WHEN -3 THEN -1 /* ExpectedVersion.NoStream */ + WHEN -1 THEN -1 /* ExpectedVersion.Empty */ + ELSE _expected_version END), __schema__.streams.position, __schema__.streams.id_internal + INTO _current_version, _current_position, _stream_id_internal + FROM __schema__.streams + WHERE __schema__.streams.id = _stream_id; + + IF (_expected_version >= 0 AND ( + SELECT __schema__.streams.version + FROM __schema__.streams + WHERE __schema__.streams.id_internal = _stream_id_internal + ) < _expected_version) + THEN + RAISE EXCEPTION 'WrongExpectedVersion'; + END IF; + + IF (_expected_version >= 0 AND _stream_id_internal IS NULL) + THEN + RAISE EXCEPTION 'WrongExpectedVersion'; + END IF; + + IF cardinality(_new_stream_messages) > 0 + THEN + INSERT INTO __schema__.messages (message_id, + stream_id_internal, + stream_version, + created_utc, + type, + json_data, + json_metadata) + SELECT m.message_id, _stream_id_internal, _current_version + (row_number() + over ()) :: int, _created_utc, m.type, m.json_data, m.json_metadata + FROM unnest(_new_stream_messages) m + ON CONFLICT DO NOTHING; + GET DIAGNOSTICS _success = ROW_COUNT; + + IF (_success <> cardinality(_new_stream_messages)) + THEN + IF (_expected_version = -2) /* ExpectedVersion.Any */ + THEN + + PERFORM __schema__.enforce_idempotent_append( + _stream_id, + _current_version + 1 - _success, + false, + _new_stream_messages); + + ELSEIF _expected_version = -3 /* ExpectedVersion.NoStream */ + THEN + RAISE EXCEPTION 'WhyAreYouHere'; /* there is no way to get here? */ + ELSEIF _expected_version = -1 /* ExpectedVersion.Empty */ + THEN + PERFORM __schema__.enforce_idempotent_append( + _stream_id, + 0, + false, + _new_stream_messages); + ELSE + PERFORM __schema__.enforce_idempotent_append( + _stream_id, + _expected_version + 1 - _success, + true, + _new_stream_messages); + SELECT version, position, id_internal + INTO _current_version, _current_position, _stream_id_internal + FROM __schema__.streams + WHERE __schema__.streams.id = _stream_id; + + RETURN QUERY + SELECT _current_version, _current_position; + RETURN; + END IF; + END IF; + + SELECT COALESCE(__schema__.messages.position, -1), COALESCE(__schema__.messages.stream_version, -1) + INTO _current_position, _current_version + FROM __schema__.messages + WHERE __schema__.messages.stream_id_internal = _stream_id_internal + ORDER BY __schema__.messages.position DESC + LIMIT 1; + + UPDATE __schema__.streams + SET "version" = _current_version, + "position" = _current_position + WHERE id_internal = _stream_id_internal; + + RETURN QUERY + SELECT _current_version, _current_position; + + ELSE + RETURN QUERY + SELECT -1, -1 :: BIGINT; + + END IF; + +END; + +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/DeleteStream.sql b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/DeleteStream.sql new file mode 100644 index 000000000..9ec69ee94 --- /dev/null +++ b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/DeleteStream.sql @@ -0,0 +1,73 @@ +CREATE OR REPLACE FUNCTION __schema__.delete_stream( + _stream_id CHAR(42), + _expected_version INT, + _created_utc TIMESTAMP WITH TIME ZONE, + _deletion_tracking_disabled BOOLEAN, + _deleted_stream_id CHAR(42), + _deleted_stream_id_original VARCHAR(1000), + _deleted_stream_message __schema__.new_stream_message) + RETURNS VOID +AS $F$ +DECLARE + _stream_id_internal INT; + _latest_stream_version INT; + _affected INT; +BEGIN + IF _created_utc IS NULL + THEN + _created_utc = now() at time zone 'utc'; + END IF; + SELECT __schema__.streams.id_internal + INTO _stream_id_internal + FROM __schema__.streams + WHERE __schema__.streams.id = _stream_id; + + IF _expected_version = -1 /* ExpectedVersion.NoStream */ + THEN + RAISE EXCEPTION 'WrongExpectedVersion'; + ELSIF _expected_version >= 0 /* ExpectedVersion */ + THEN + IF _stream_id_internal IS NULL + THEN + RAISE EXCEPTION 'WrongExpectedVersion'; + END IF; + + SELECT __schema__.messages.stream_version + INTO _latest_stream_version + FROM __schema__.messages + WHERE __schema__.messages.stream_id_internal = _stream_id_internal + ORDER BY __schema__.messages.position DESC + LIMIT 1; + + IF _latest_stream_version != _expected_version + THEN + RAISE EXCEPTION 'WrongExpectedVersion'; + END IF; + END IF; + + DELETE FROM __schema__.messages WHERE __schema__.messages.stream_id_internal = _stream_id_internal; + + DELETE FROM __schema__.streams WHERE __schema__.streams.id = _stream_id; + + IF (_deletion_tracking_disabled = TRUE) + THEN + RETURN; + END IF; + + GET DIAGNOSTICS _affected = ROW_COUNT; + + IF _affected > 0 + THEN + PERFORM __schema__.append_to_stream( + _deleted_stream_id, + _deleted_stream_id_original, + NULL, + -2, + _created_utc, + ARRAY [_deleted_stream_message]); + + END IF; +END; + +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/DeleteStreamMessages.sql b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/DeleteStreamMessages.sql new file mode 100644 index 000000000..1ad2b0861 --- /dev/null +++ b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/DeleteStreamMessages.sql @@ -0,0 +1,46 @@ +CREATE OR REPLACE FUNCTION __schema__.delete_stream_messages( + _stream_id CHAR(42), + _message_ids UUID [], + _deletion_tracking_disabled BOOLEAN, + _deleted_stream_id CHAR(42), + _deleted_stream_id_original VARCHAR(1000), + _created_utc TIMESTAMP WITH TIME ZONE, + _deleted_messages __schema__.new_stream_message [] +) + RETURNS VOID +AS $F$ +DECLARE + _stream_id_internal INT; + _deleted_count NUMERIC; +BEGIN + IF _created_utc IS NULL THEN + _created_utc = now() at time zone 'utc'; + END IF; + + SELECT __schema__.streams.id_internal + INTO _stream_id_internal + FROM __schema__.streams + WHERE __schema__.streams.id = _stream_id; + + WITH deleted AS (DELETE FROM __schema__.messages + WHERE __schema__.messages.stream_id_internal = _stream_id_internal + AND __schema__.messages.message_id = ANY (_message_ids) + RETURNING *) + SELECT count(*) + FROM deleted + INTO _deleted_count; + + IF (_deletion_tracking_disabled = FALSE AND _deleted_count > 0) + THEN + PERFORM __schema__.append_to_stream( + _deleted_stream_id, + _deleted_stream_id_original, + NULL, + -2, + _created_utc, + _deleted_messages); + END IF; +END; + +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/DropAll.sql b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/DropAll.sql new file mode 100644 index 000000000..aa6c25354 --- /dev/null +++ b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/DropAll.sql @@ -0,0 +1,78 @@ +DROP FUNCTION IF EXISTS __schema__.append_to_stream( + CHAR(42), + VARCHAR(1000), + INT, + TIMESTAMP WITH TIME ZONE, + __schema__ .new_stream_message [] +) CASCADE; +DROP FUNCTION IF EXISTS __schema__.delete_stream( + CHAR(42), + INT, + TIMESTAMP WITH TIME ZONE, + CHAR(42), + VARCHAR(1000), + __schema__ .new_stream_message +) CASCADE; +DROP FUNCTION IF EXISTS __schema__.delete_stream_messages( + CHAR(42), + UUID [], + CHAR(42), + VARCHAR(1000), + TIMESTAMP WITH TIME ZONE, + __schema__ .new_stream_message [] +) CASCADE; +DROP FUNCTION IF EXISTS __schema__.enforce_idempotent_append( + CHAR(42), + INT, + BOOLEAN, + __schema__ .new_stream_message [] +) CASCADE; +DROP FUNCTION IF EXISTS __schema__.list_streams( + INT, + INT +) CASCADE; +DROP FUNCTION IF EXISTS __schema__.list_streams_starting_with( + VARCHAR(1000), + INT, + INT +) CASCADE; +DROP FUNCTION IF EXISTS __schema__.list_streams_ending_with( + VARCHAR(1000), + INT, + INT +) CASCADE; +DROP FUNCTION IF EXISTS __schema__.read( + CHAR(42), + INT, + INT, + BOOLEAN, + BOOLEAN +) CASCADE; +DROP FUNCTION IF EXISTS __schema__.read_all( + INT, + BIGINT, + BOOLEAN, + BOOLEAN +) CASCADE; +DROP FUNCTION IF EXISTS __schema__.read_schema_version() CASCADE; +DROP FUNCTION IF EXISTS __schema__.read_head_position() CASCADE; +DROP FUNCTION IF EXISTS __schema__.read_json_data( + CHAR(42), + INT +) CASCADE; +DROP FUNCTION IF EXISTS __schema__.read_stream_message_before_created_count( + CHAR(42), + TIMESTAMP WITH TIME ZONE +) CASCADE; +DROP FUNCTION IF EXISTS __schema__.read_stream_version_of_message_id( + INT, + UUID +); +DROP FUNCTION IF EXISTS __schema__.scavenge( + CHAR(42) +) CASCADE; +DROP TABLE IF EXISTS __schema__.messages CASCADE; +DROP SEQUENCE IF EXISTS __schema__.messages_seq CASCADE; +DROP TABLE IF EXISTS __schema__.streams CASCADE; +DROP SEQUENCE IF EXISTS __schema__.streams_seq CASCADE; +DROP TYPE IF EXISTS __schema__.new_stream_message CASCADE; \ No newline at end of file diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/EnableExplainAnalyze.sql b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/EnableExplainAnalyze.sql new file mode 100644 index 000000000..450617841 --- /dev/null +++ b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/EnableExplainAnalyze.sql @@ -0,0 +1,7 @@ +LOAD 'auto_explain'; +SET auto_explain.log_min_duration = 0; +SET auto_explain.log_nested_statements = ON; +SET auto_explain.log_analyze = ON; +SET auto_explain.log_format = TEXT; +SET auto_explain.log_buffers = ON; +SET auto_explain.log_verbose = ON; diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/EnforceIdempotentAppend.sql b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/EnforceIdempotentAppend.sql new file mode 100644 index 000000000..e0602a24e --- /dev/null +++ b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/EnforceIdempotentAppend.sql @@ -0,0 +1,38 @@ +CREATE OR REPLACE FUNCTION __schema__.enforce_idempotent_append( + _stream_id CHAR(42), + _start INT, + _check_length BOOLEAN, + _new_stream_messages __schema__.new_stream_message []) + RETURNS VOID AS $F$ +DECLARE + _message_id_record RECORD; + _message_id_cursor REFCURSOR; + _message_ids UUID [] = '{}' :: UUID []; +BEGIN + _message_id_cursor = (SELECT * + FROM __schema__.read(_stream_id, cardinality(_new_stream_messages), _start, true, false) + OFFSET 1); + + FETCH FROM _message_id_cursor + INTO _message_id_record; + + WHILE FOUND LOOP + _message_ids = array_append(_message_ids, _message_id_record.message_id); + + FETCH FROM _message_id_cursor + INTO _message_id_record; + END LOOP; + + IF (_check_length AND cardinality(_new_stream_messages) > cardinality(_message_ids)) + THEN + RAISE EXCEPTION 'WrongExpectedVersion' + USING HINT = 'Wrong message count'; + END IF; + + IF _message_ids <> (SELECT ARRAY(SELECT n.message_id FROM unnest(_new_stream_messages) n)) + THEN + RAISE EXCEPTION 'WrongExpectedVersion'; + END IF; +END; +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ListStreams.sql b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ListStreams.sql new file mode 100644 index 000000000..22a2c0650 --- /dev/null +++ b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ListStreams.sql @@ -0,0 +1,19 @@ +CREATE OR REPLACE FUNCTION __schema__.list_streams( + _max_count INT, + _after_id_internal INT +) + RETURNS TABLE( + stream_id VARCHAR(1000), + id_internal INT + ) AS $F$ +BEGIN + RETURN QUERY + SELECT __schema__.streams.id_original, __schema__.streams.id_internal + FROM __schema__.streams + WHERE __schema__.streams.id_internal > _after_id_internal + ORDER BY __schema__.streams.id_internal ASC + LIMIT _max_count; +END; + +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ListStreamsEndingWith.sql b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ListStreamsEndingWith.sql new file mode 100644 index 000000000..b9e6322d8 --- /dev/null +++ b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ListStreamsEndingWith.sql @@ -0,0 +1,21 @@ +CREATE OR REPLACE FUNCTION __schema__.list_streams_ending_with( + _pattern VARCHAR(1000), + _max_count INT, + _after_id_internal INT +) + RETURNS TABLE( + stream_id VARCHAR(1000), + id_internal INT + ) AS $F$ +BEGIN + RETURN QUERY + SELECT __schema__.streams.id_original, __schema__.streams.id_internal + FROM __schema__.streams + WHERE REVERSE(__schema__.streams.id_original) LIKE CONCAT(REVERSE(_pattern), '%') + AND __schema__.streams.id_internal > _after_id_internal + ORDER BY __schema__.streams.id_internal ASC + LIMIT _max_count; +END; + +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ListStreamsStartingWith.sql b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ListStreamsStartingWith.sql new file mode 100644 index 000000000..4a93e5cd8 --- /dev/null +++ b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ListStreamsStartingWith.sql @@ -0,0 +1,21 @@ +CREATE OR REPLACE FUNCTION __schema__.list_streams_starting_with( + _pattern VARCHAR(1000), + _max_count INT, + _after_id_internal INT +) + RETURNS TABLE( + stream_id VARCHAR(1000), + id_internal INT + ) AS $F$ +BEGIN + RETURN QUERY + SELECT __schema__.streams.id_original, __schema__.streams.id_internal + FROM __schema__.streams + WHERE __schema__.streams.id_original LIKE CONCAT(_pattern, '%') + AND __schema__.streams.id_internal > _after_id_internal + ORDER BY __schema__.streams.id_internal ASC + LIMIT _max_count; +END; + +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Migration.sql b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Migration.sql new file mode 100644 index 000000000..6adeb62ae --- /dev/null +++ b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Migration.sql @@ -0,0 +1,3 @@ +SET TimeZone='UTC'; +ALTER TABLE __schema__.messages ALTER COLUMN "created_utc" TYPE TIMESTAMP WITH TIME ZONE; +COMMENT ON SCHEMA __schema__ IS '{ "version": 2 }'; \ No newline at end of file diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Parameters.cs b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Parameters.cs new file mode 100644 index 000000000..a77c065c5 --- /dev/null +++ b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Parameters.cs @@ -0,0 +1,272 @@ +namespace SqlStreamStore.PgSqlScriptsV2 +{ + using System; + using Npgsql; + using NpgsqlTypes; + using SqlStreamStore.Infrastructure; + using SqlStreamStore.Streams; + using StreamStoreStore.Json; + + internal static class Parameters + { + private const int StreamIdSize = 42; + private const int OriginalStreamIdSize = 1000; + + public static NpgsqlParameter DeletedStreamId => new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Char, + Size = StreamIdSize, + TypedValue = PostgresqlStreamId.Deleted.Id + }; + + public static NpgsqlParameter DeletedStreamIdOriginal => new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Varchar, + Size = OriginalStreamIdSize, + TypedValue = PostgresqlStreamId.Deleted.IdOriginal + }; + + public static NpgsqlParameter StreamId(PostgresqlStreamId value) + { + return new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Char, + Size = StreamIdSize, + TypedValue = value.Id + }; + } + + public static NpgsqlParameter StreamIdOriginal(PostgresqlStreamId value) + { + return new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Varchar, + Size = OriginalStreamIdSize, + TypedValue = value.IdOriginal + }; + } + + public static NpgsqlParameter MetadataStreamId(PostgresqlStreamId value) + { + return new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Char, + Size = StreamIdSize, + TypedValue = value.Id + }; + } + + public static NpgsqlParameter MetadataStreamIdOriginal(PostgresqlStreamId value) + { + return new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Char, + Size = StreamIdSize, + TypedValue = value.IdOriginal + }; + } + + public static NpgsqlParameter DeletedMessages(PostgresqlStreamId streamId, params Guid[] messageIds) + { + return new NpgsqlParameter + { + TypedValue = Array.ConvertAll( + messageIds, + messageId => PostgresNewStreamMessage.FromNewStreamMessage( + Deleted.CreateMessageDeletedMessage(streamId.IdOriginal, messageId)) + ) + }; + } + + public static NpgsqlParameter DeletedStreamMessage(PostgresqlStreamId streamId) + { + return new NpgsqlParameter + { + TypedValue = PostgresNewStreamMessage.FromNewStreamMessage( + Deleted.CreateStreamDeletedMessage(streamId.IdOriginal)) + }; + } + + public static NpgsqlParameter ExpectedVersion(int value) + { + return new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Integer, + TypedValue = value + }; + } + + public static NpgsqlParameter CreatedUtc(DateTime? value) + { + return value.HasValue + ? (NpgsqlParameter) new NpgsqlParameter + { + TypedValue = value.Value, + NpgsqlDbType = NpgsqlDbType.TimestampTz + } + : new NpgsqlParameter + { + TypedValue = DBNull.Value, + NpgsqlDbType = NpgsqlDbType.TimestampTz + }; + } + + public static NpgsqlParameter NewStreamMessages(NewStreamMessage[] value) + { + return new NpgsqlParameter + { + TypedValue = Array.ConvertAll(value, PostgresNewStreamMessage.FromNewStreamMessage) + }; + } + + public static NpgsqlParameter MetadataStreamMessage( + PostgresqlStreamId streamId, + int expectedVersion, + MetadataMessage value) + { + var jsonData = SimpleJson.SerializeObject(value); + return new NpgsqlParameter + { + TypedValue = PostgresNewStreamMessage.FromNewStreamMessage( + new NewStreamMessage( + MetadataMessageIdGenerator.Create(streamId.IdOriginal, expectedVersion, jsonData), + "$stream-metadata", + jsonData)) + }; + } + + public static NpgsqlParameter Count(int value) + { + return new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Integer, + TypedValue = value + }; + } + + public static NpgsqlParameter Version(int value) + { + return new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Integer, + TypedValue = value + }; + } + + public static NpgsqlParameter ReadDirection(ReadDirection direction) + { + return new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Boolean, + TypedValue = direction == Streams.ReadDirection.Forward + }; + } + + public static NpgsqlParameter Prefetch(bool value) + { + return new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Boolean, + TypedValue = value + }; + } + + public static NpgsqlParameter MessageIds(Guid[] value) + { + return new NpgsqlParameter + { + Value = value + }; + } + + public static NpgsqlParameter Position(long value) + { + return new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Bigint, + TypedValue = value + }; + } + + public static NpgsqlParameter OptionalMaxAge(int? value) + { + return new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Integer, + NpgsqlValue = value.HasValue ? (object) value.Value : DBNull.Value + }; + } + + public static NpgsqlParameter OptionalMaxCount(int? value) + { + return new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Integer, + NpgsqlValue = value.HasValue ? (object) value.Value : DBNull.Value + }; + } + + public static NpgsqlParameter MaxCount(int value) + { + return new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Integer, + TypedValue = value + }; + } + + public static NpgsqlParameter OptionalStartingAt(int? value) + { + return value.HasValue + ? (NpgsqlParameter) new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Integer, + TypedValue = value.Value + } + : new NpgsqlParameter + { + TypedValue = DBNull.Value + }; + } + + public static NpgsqlParameter OptionalAfterIdInternal(int? value) + { + return value.HasValue + ? (NpgsqlParameter) new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Integer, + TypedValue = value.Value + } + : new NpgsqlParameter + { + TypedValue = DBNull.Value + }; + } + + public static NpgsqlParameter Pattern(string value) + { + return new NpgsqlParameter + { + TypedValue = value, + NpgsqlDbType = NpgsqlDbType.Varchar + }; + } + + public static NpgsqlParameter DeletionTrackingDisabled(bool deletionTrackingDisabled) + { + return new NpgsqlParameter + { + TypedValue = deletionTrackingDisabled, + NpgsqlDbType = NpgsqlDbType.Boolean + }; + } + + public static NpgsqlParameter Empty() + { + return new NpgsqlParameter + { + TypedValue = DBNull.Value + }; + } + } +} \ No newline at end of file diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Read.sql b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Read.sql new file mode 100644 index 000000000..3dee4eb07 --- /dev/null +++ b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Read.sql @@ -0,0 +1,53 @@ +CREATE OR REPLACE FUNCTION __schema__.read( + _stream_id CHAR(42), + _count INT, + _version INT, + _forwards BOOLEAN, + _prefetch BOOLEAN +) + RETURNS SETOF REFCURSOR +AS $F$ +DECLARE + _stream_id_internal INT; + _stream_info REFCURSOR := 'stream_info'; + _messages REFCURSOR := 'messages'; +BEGIN + SELECT __schema__.streams.id_internal + INTO _stream_id_internal + FROM __schema__.streams + WHERE __schema__.streams.id = _stream_id; + + OPEN _stream_info FOR + SELECT __schema__.streams.version as stream_version, + __schema__.streams.position as position, + __schema__.streams.max_age as max_age + FROM __schema__.streams + WHERE __schema__.streams.id_internal = _stream_id_internal; + + RETURN NEXT _stream_info; + + OPEN _messages FOR + SELECT __schema__.streams.id_original AS stream_id, + __schema__.messages.message_id, + __schema__.messages.stream_version, + __schema__.messages.position, + __schema__.messages.created_utc, + __schema__.messages.type, + __schema__.messages.json_metadata, + (CASE _prefetch + WHEN TRUE THEN __schema__.messages.json_data + ELSE NULL END) + FROM __schema__.messages + INNER JOIN __schema__.streams ON __schema__.messages.stream_id_internal = __schema__.streams.id_internal + WHERE (CASE + WHEN _forwards THEN __schema__.messages.stream_version >= _version AND id_internal = _stream_id_internal + ELSE __schema__.messages.stream_version <= _version AND id_internal = _stream_id_internal END) + ORDER BY (CASE + WHEN _forwards THEN __schema__.messages.stream_version + ELSE -__schema__.messages.stream_version END) + LIMIT _count; + + RETURN NEXT _messages; +END; +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadAll.sql b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadAll.sql new file mode 100644 index 000000000..93e4b63ac --- /dev/null +++ b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadAll.sql @@ -0,0 +1,47 @@ +CREATE OR REPLACE FUNCTION __schema__.read_all( + _count INT, + _position BIGINT, + _forwards BOOLEAN, + _prefetch BOOLEAN +) + RETURNS TABLE( + stream_id VARCHAR(1000), + message_id UUID, + stream_version INT, + "position" BIGINT, + create_utc TIMESTAMP WITH TIME ZONE, + "type" VARCHAR(128), + json_metadata JSONB, + json_data JSONB, + max_age INT + ) +AS $F$ +BEGIN + + RETURN QUERY + WITH messages AS ( + SELECT __schema__.streams.id_original, + __schema__.messages.message_id, + __schema__.messages.stream_version, + __schema__.messages.position, + __schema__.messages.created_utc, + __schema__.messages.type, + __schema__.messages.json_metadata, + (CASE _prefetch + WHEN TRUE THEN __schema__.messages.json_data + ELSE NULL END), + __schema__.streams.max_age + FROM __schema__.messages + INNER JOIN __schema__.streams ON __schema__.messages.stream_id_internal = __schema__.streams.id_internal + WHERE (CASE + WHEN _forwards THEN __schema__.messages.position >= _position + ELSE __schema__.messages.position <= _position END) + ORDER BY + (CASE WHEN _forwards THEN __schema__.messages.position END), + (CASE WHEN not _forwards THEN __schema__.messages.position END) DESC + LIMIT _count + ) + SELECT * FROM messages LIMIT _count; +END; +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadHeadPosition.sql b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadHeadPosition.sql new file mode 100644 index 000000000..5eddb47c8 --- /dev/null +++ b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadHeadPosition.sql @@ -0,0 +1,8 @@ +CREATE OR REPLACE FUNCTION __schema__.read_head_position() + RETURNS BIGINT +AS $F$ +BEGIN + RETURN (SELECT max(__schema__.messages.position) FROM __schema__.messages); +END; +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadJsonData.sql b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadJsonData.sql new file mode 100644 index 000000000..1b9e3185e --- /dev/null +++ b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadJsonData.sql @@ -0,0 +1,16 @@ +CREATE OR REPLACE FUNCTION __schema__.read_json_data( + _stream_id CHAR(42), + _stream_version INT +) + RETURNS VARCHAR +AS $F$ +BEGIN + RETURN (SELECT __schema__.messages.json_data + FROM __schema__.messages + JOIN __schema__.streams ON __schema__.messages.stream_id_internal = __schema__.streams.id_internal + WHERE __schema__.messages.stream_version = _stream_version + AND __schema__.streams.id = _stream_id + LIMIT 1); +END; +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadSchemaVersion.sql b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadSchemaVersion.sql new file mode 100644 index 000000000..dba135e67 --- /dev/null +++ b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadSchemaVersion.sql @@ -0,0 +1,13 @@ +CREATE OR REPLACE FUNCTION __schema__.read_schema_version() + RETURNS INT +AS $F$ +BEGIN + + RETURN ( + SELECT obj_description :: JSON -> 'version' + FROM obj_description('__schema__' :: REGNAMESPACE, 'pg_namespace') + ); + +END; +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadStreamHeadPosition.sql b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadStreamHeadPosition.sql new file mode 100644 index 000000000..cefe0d638 --- /dev/null +++ b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadStreamHeadPosition.sql @@ -0,0 +1,10 @@ +CREATE OR REPLACE FUNCTION __schema__.read_stream_head_position( + _stream_id CHAR(42) +) + RETURNS BIGINT +AS $F$ +BEGIN + RETURN (SELECT (__schema__.streams.position) FROM __schema__.streams WHERE __schema__.streams.id = _stream_id); +END; +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadStreamHeadVersion.sql b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadStreamHeadVersion.sql new file mode 100644 index 000000000..56c82a806 --- /dev/null +++ b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadStreamHeadVersion.sql @@ -0,0 +1,10 @@ +CREATE OR REPLACE FUNCTION __schema__.read_stream_head_version( + _stream_id CHAR(42) +) + RETURNS INT +AS $F$ +BEGIN + RETURN (SELECT (__schema__.streams.version) FROM __schema__.streams WHERE __schema__.streams.id = _stream_id); +END; +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadStreamVersionOfMessageId.sql b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadStreamVersionOfMessageId.sql new file mode 100644 index 000000000..6921015dd --- /dev/null +++ b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadStreamVersionOfMessageId.sql @@ -0,0 +1,14 @@ +CREATE OR REPLACE FUNCTION __schema__.read_stream_version_of_message_id( + _stream_id_internal INT, + _message_id UUID +) + RETURNS INT +AS $F$ +BEGIN + RETURN (SELECT __schema__.messages.stream_version + FROM __schema__.messages + WHERE __schema__.messages.message_id = _message_id + AND __schema__.messages.stream_id_internal = _stream_id_internal); +END; +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Scavenge.sql b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Scavenge.sql new file mode 100644 index 000000000..686c65b0d --- /dev/null +++ b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Scavenge.sql @@ -0,0 +1,33 @@ +CREATE OR REPLACE FUNCTION __schema__.scavenge( + _stream_id CHAR(42) +) + RETURNS TABLE( + message_id UUID + ) AS $F$ +DECLARE + _stream_id_internal INT; + _max_count INT; +BEGIN + SELECT __schema__.streams.id_internal, __schema__.streams.max_count + INTO _stream_id_internal, _max_count + FROM __schema__.streams + WHERE __schema__.streams.id = _stream_id; + + IF (_max_count IS NULL) + THEN RETURN; + END IF; + + RETURN QUERY + SELECT __schema__.messages.message_id + FROM __schema__.messages + WHERE __schema__.messages.stream_id_internal = _stream_id_internal + AND __schema__.messages.message_id NOT IN(SELECT __schema__.messages.message_id + FROM __schema__.messages + WHERE __schema__.messages.stream_id_internal = _stream_id_internal + ORDER BY __schema__.messages.stream_version desc + LIMIT _max_count) + ORDER BY __schema__.messages.stream_version; + +END; +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Schema.cs b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Schema.cs new file mode 100644 index 000000000..0588c55ae --- /dev/null +++ b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Schema.cs @@ -0,0 +1,40 @@ +namespace SqlStreamStore.PgSqlScriptsV2 +{ + internal class Schema + { + private readonly string _schema; + private readonly Scripts _scripts; + + public string Definition => _scripts.CreateSchema; + public string Migration => _scripts.Migration; + public string DropAll => _scripts.DropAll; + public string EnableExplainAnalyze => _scripts.EnableExplainAnalyze; + + public string NewStreamMessage => $"{_schema}.new_stream_message"; + + public string AppendToStream => $"{_schema}.append_to_stream"; + public string Scavenge => $"{_schema}.scavenge"; + public string SetStreamMetadata => $"{_schema}.set_stream_metadata"; + public string DeleteStream => $"{_schema}.delete_stream"; + public string DeleteStreamMessages => $"{_schema}.delete_stream_messages"; + public string ListStreams => $"{_schema}.list_streams"; + public string ListStreamsStartingWith => $"{_schema}.list_streams_starting_with"; + public string ListStreamsEndingWith => $"{_schema}.list_streams_ending_with"; + public string Read => $"{_schema}.read"; + public string ReadAll => $"{_schema}.read_all"; + public string ReadAllHeadPosition => $"{_schema}.read_head_position"; + public string ReadStreamHeadPosition => $"{_schema}.read_stream_head_position"; + public string ReadStreamHeadVersion => $"{_schema}.read_stream_head_version"; + public string ReadJsonData => $"{_schema}.read_json_data"; + public string ReadSchemaVersion => $"{_schema}.read_schema_version"; + public string ReadStreamMessageBeforeCreatedCount => $"{_schema}.read_stream_message_before_created_count"; + + public static string FetchAll(string refcursor) => $@"FETCH ALL IN ""{refcursor}"";"; + + public Schema(string schema) + { + _schema = schema; + _scripts = new Scripts(schema); + } + } +} \ No newline at end of file diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Scripts.cs b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Scripts.cs new file mode 100644 index 000000000..0ca644d41 --- /dev/null +++ b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Scripts.cs @@ -0,0 +1,100 @@ +namespace SqlStreamStore.PgSqlScriptsV2 +{ + using System; + using System.Collections.Concurrent; + using System.IO; + using System.Reflection; + + internal class Scripts + { + private static readonly Assembly s_assembly = typeof(Scripts) + .GetTypeInfo() + .Assembly; + + private readonly string _schema; + + private readonly ConcurrentDictionary _scripts + = new ConcurrentDictionary(); + + public Scripts(string schema) + { + _schema = schema; + } + + public string DropAll => GetScript(nameof(DropAll)); + + public string EnableExplainAnalyze => GetScript(nameof(EnableExplainAnalyze)); + + private string Tables => GetScript(nameof(Tables)); + + private string AppendToStream => GetScript(nameof(AppendToStream)); + private string DeleteStream => GetScript(nameof(DeleteStream)); + + private string DeleteStreamMessages => GetScript(nameof(DeleteStreamMessages)); + private string EnforceIdempotentAppend => GetScript(nameof(EnforceIdempotentAppend)); + private string ListStreams => GetScript(nameof(ListStreams)); + private string ListStreamsStartingWith => GetScript(nameof(ListStreamsStartingWith)); + private string ListStreamsEndingWith => GetScript(nameof(ListStreamsEndingWith)); + private string ReadAll => GetScript(nameof(ReadAll)); + + private string Read => GetScript(nameof(Read)); + + private string ReadJsonData => GetScript(nameof(ReadJsonData)); + + private string ReadHeadPosition => GetScript(nameof(ReadHeadPosition)); + + private string ReadStreamHeadPosition => GetScript(nameof(ReadStreamHeadPosition)); + + private string ReadStreamHeadVersion => GetScript(nameof(ReadStreamHeadVersion)); + + private string ReadSchemaVersion => GetScript(nameof(ReadSchemaVersion)); + + private string ReadStreamVersionOfMessageId => GetScript(nameof(ReadStreamVersionOfMessageId)); + + private string Scavenge => GetScript(nameof(Scavenge)); + + private string SetStreamMetadata => GetScript(nameof(SetStreamMetadata)); + + public string Migration => GetScript(nameof(Migration)); + + public string CreateSchema => string.Join( + Environment.NewLine, + Tables, + AppendToStream, + DeleteStream, + DeleteStreamMessages, + EnforceIdempotentAppend, + ListStreams, + ListStreamsStartingWith, + ListStreamsEndingWith, + Read, + ReadAll, + ReadJsonData, + ReadHeadPosition, + ReadStreamHeadPosition, + ReadStreamHeadVersion, + ReadSchemaVersion, + ReadStreamVersionOfMessageId, + Scavenge, + SetStreamMetadata); + + private string GetScript(string name) => _scripts.GetOrAdd(name, + key => + { + using(var stream = s_assembly.GetManifestResourceStream(typeof(Scripts), $"{key}.sql")) + { + if(stream == null) + { + throw new Exception($"Embedded resource, {name}, not found. BUG!"); + } + + using(StreamReader reader = new StreamReader(stream)) + { + return reader + .ReadToEnd() + .Replace("__schema__", _schema); + } + } + }); + } +} \ No newline at end of file diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/SetStreamMetadata.sql b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/SetStreamMetadata.sql new file mode 100644 index 000000000..fe1417baf --- /dev/null +++ b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/SetStreamMetadata.sql @@ -0,0 +1,38 @@ +CREATE OR REPLACE FUNCTION __schema__.set_stream_metadata( + _stream_id CHAR(42), + _metadata_stream_id CHAR(42), + _metadata_stream_id_original CHAR(42), + _max_age INT, + _max_count INT, + _expected_version INT, + _created_utc TIMESTAMP WITH TIME ZONE, + _metadata_message __schema__.new_stream_message) + RETURNS INT AS $F$ +DECLARE + _current_version INT; +BEGIN + IF _created_utc IS NULL + THEN + _created_utc = now() at time zone 'utc'; + END IF; + + SELECT current_version + FROM __schema__.append_to_stream( + _metadata_stream_id, + _metadata_stream_id_original, + NULL, + _expected_version, + _created_utc, + ARRAY [_metadata_message] + ) + INTO _current_version; + + UPDATE __schema__.streams + SET max_age = _max_age, + max_count = _max_count + WHERE __schema__.streams.id = _stream_id; + + RETURN _current_version; +END; +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Tables.sql b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Tables.sql new file mode 100644 index 000000000..e8605a905 --- /dev/null +++ b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Tables.sql @@ -0,0 +1,62 @@ +CREATE SEQUENCE IF NOT EXISTS __schema__.streams_seq + START 1; + +CREATE TABLE IF NOT EXISTS __schema__.streams ( + id CHAR(42) NOT NULL, + id_original VARCHAR(1000) NOT NULL, + id_internal INT NOT NULL DEFAULT nextval('__schema__.streams_seq'), + version INT NOT NULL DEFAULT (-1), + position BIGINT NOT NULL DEFAULT (-1), + max_age INT NULL, + max_count INT NULL, + CONSTRAINT pk_streams PRIMARY KEY (id_internal), + CONSTRAINT uq_streams_id UNIQUE (id), + CONSTRAINT ck_version_gte_negative_one CHECK (version >= -1) +); + +CREATE INDEX IF NOT EXISTS ix_id_original + ON __schema__.streams (id_original); + +CREATE INDEX IF NOT EXISTS ix_id_original_reversed + ON __schema__.streams (REVERSE(id_original)); + +COMMENT ON SCHEMA __schema__ +IS '{ "version": 2 }'; + +ALTER SEQUENCE __schema__.streams_seq + OWNED BY __schema__.streams.id_internal; + +CREATE SEQUENCE IF NOT EXISTS __schema__.messages_seq + START 0 + MINVALUE 0; + +CREATE TABLE IF NOT EXISTS __schema__.messages ( + stream_id_internal INT NOT NULL, + stream_version INT NOT NULL, + "position" BIGINT NOT NULL DEFAULT nextval('__schema__.messages_seq'), + message_id UUID NOT NULL, + created_utc TIMESTAMP WITH TIME ZONE NOT NULL, + type VARCHAR(128) NOT NULL, + json_data JSONB NOT NULL, + json_metadata JSONB, + CONSTRAINT pk_messages PRIMARY KEY (position), + CONSTRAINT fk_messages_stream FOREIGN KEY (stream_id_internal) REFERENCES __schema__.streams (id_internal), + CONSTRAINT uq_messages_stream_id_internal_and_stream_version UNIQUE (stream_id_internal, stream_version), + CONSTRAINT uq_stream_id_internal_and_message_id UNIQUE (stream_id_internal, message_id), + CONSTRAINT ck_stream_version_gte_zero CHECK (stream_version >= 0) +); + +ALTER SEQUENCE __schema__.messages_seq + OWNED BY __schema__.messages.position; + +DO $F$ +BEGIN + CREATE TYPE __schema__.new_stream_message AS ( + message_id UUID, + "type" VARCHAR(128), + json_data JSONB, + json_metadata JSONB); + EXCEPTION + WHEN duplicate_object + THEN null; +END $F$; \ No newline at end of file diff --git a/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Append.cs b/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Append.cs new file mode 100644 index 000000000..5fab240a8 --- /dev/null +++ b/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Append.cs @@ -0,0 +1,91 @@ +namespace SqlStreamStore +{ + using System; + using System.Runtime.ExceptionServices; + using System.Threading; + using System.Threading.Tasks; + using Npgsql; + using SqlStreamStore.PgSqlScriptsV2; + using SqlStreamStore.Streams; + + partial class PostgresStreamStoreV2 + { + protected override async Task AppendToStreamInternal( + string streamId, + int expectedVersion, + NewStreamMessage[] messages, + CancellationToken cancellationToken) + { + int maxRetries = 2; //TODO too much? too little? configurable? + Exception exception; + + int retryCount = 0; + do + { + try + { + AppendResult result; + var streamIdInfo = new StreamIdInfo(streamId); + + using(var connection = await OpenConnection(cancellationToken)) + using(var transaction = connection.BeginTransaction()) + using(var command = BuildFunctionCommand( + _schema.AppendToStream, + transaction, + Parameters.StreamId(streamIdInfo.PostgresqlStreamId), + Parameters.StreamIdOriginal(streamIdInfo.PostgresqlStreamId), + Parameters.MetadataStreamId(streamIdInfo.MetadataPosgresqlStreamId), + Parameters.ExpectedVersion(expectedVersion), + Parameters.CreatedUtc(_settings.GetUtcNow?.Invoke()), + Parameters.NewStreamMessages(messages))) + { + try + { + using(var reader = await command + .ExecuteReaderAsync(cancellationToken) + .ConfigureAwait(false)) + { + await reader.ReadAsync(cancellationToken).ConfigureAwait(false); + + result = new AppendResult(reader.GetInt32(0), reader.GetInt64(1)); + } + + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + } + catch(PostgresException ex) when(ex.IsWrongExpectedVersion()) + { + await transaction.RollbackAsync(cancellationToken).ConfigureAwait(false); + + throw new WrongExpectedVersionException( + ErrorMessages.AppendFailedWrongExpectedVersion(streamIdInfo.PostgresqlStreamId.IdOriginal, expectedVersion), + streamIdInfo.PostgresqlStreamId.IdOriginal, + expectedVersion, + ex); + } + } + + if(_settings.ScavengeAsynchronously) + { +#pragma warning disable CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed + Task.Run(() => TryScavenge(streamIdInfo, cancellationToken), cancellationToken); +#pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed + } + else + { + await TryScavenge(streamIdInfo, cancellationToken).ConfigureAwait(false); + } + + return result; + } + catch(PostgresException ex) when(ex.IsDeadlock()) + { + exception = ex; + retryCount++; + } + } while(retryCount < maxRetries); + + ExceptionDispatchInfo.Capture(exception).Throw(); + return default; // never actually run + } + } +} diff --git a/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Delete.cs b/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Delete.cs new file mode 100644 index 000000000..992ea9e77 --- /dev/null +++ b/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Delete.cs @@ -0,0 +1,111 @@ +namespace SqlStreamStore +{ + using System; + using System.Threading; + using System.Threading.Tasks; + using Npgsql; + using SqlStreamStore.PgSqlScriptsV2; + using SqlStreamStore.Streams; + + public partial class PostgresStreamStoreV2 + { + protected override async Task DeleteStreamInternal( + string streamId, + int expectedVersion, + CancellationToken cancellationToken) + { + var streamIdInfo = new StreamIdInfo(streamId); + + using(var connection = await OpenConnection(cancellationToken)) + using(var transaction = connection.BeginTransaction()) + { + await DeleteStreamInternal( + streamIdInfo.PostgresqlStreamId, + expectedVersion, + transaction, + cancellationToken); + + await DeleteStreamInternal( + streamIdInfo.MetadataPosgresqlStreamId, + ExpectedVersion.Any, + transaction, + cancellationToken); + + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + } + } + + private async Task DeleteStreamInternal( + PostgresqlStreamId streamId, + int expectedVersion, + NpgsqlTransaction transaction, + CancellationToken cancellationToken) + { + using(var command = BuildFunctionCommand( + _schema.DeleteStream, + transaction, + Parameters.StreamId(streamId), + Parameters.ExpectedVersion(expectedVersion), + Parameters.CreatedUtc(_settings.GetUtcNow?.Invoke()), + Parameters.DeletionTrackingDisabled(_settings.DisableDeletionTracking), + _settings.DisableDeletionTracking ? Parameters.Empty() : Parameters.DeletedStreamId, + _settings.DisableDeletionTracking ? Parameters.Empty() : Parameters.DeletedStreamIdOriginal, + _settings.DisableDeletionTracking ? Parameters.Empty() : Parameters.DeletedStreamMessage(streamId))) + { + try + { + await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + catch(PostgresException ex) when(ex.IsWrongExpectedVersion()) + { + await transaction.RollbackAsync(cancellationToken).ConfigureAwait(false); + + throw new WrongExpectedVersionException( + ErrorMessages.DeleteStreamFailedWrongExpectedVersion(streamId.IdOriginal, expectedVersion), + streamId.IdOriginal, + expectedVersion, + ex); + } + } + } + + protected override async Task DeleteEventInternal( + string streamId, + Guid eventId, + CancellationToken cancellationToken) + { + var streamIdInfo = new StreamIdInfo(streamId); + + using(var connection = await OpenConnection(cancellationToken)) + using(var transaction = connection.BeginTransaction()) + { + await DeleteEventsInternal(streamIdInfo, new[] { eventId }, transaction, cancellationToken); + + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + } + } + + private async Task DeleteEventsInternal( + StreamIdInfo streamIdInfo, + Guid[] eventIds, + NpgsqlTransaction transaction, + CancellationToken cancellationToken) + { + using(var command = BuildFunctionCommand( + _schema.DeleteStreamMessages, + transaction, + Parameters.StreamId(streamIdInfo.PostgresqlStreamId), + Parameters.MessageIds(eventIds), + Parameters.DeletionTrackingDisabled(_settings.DisableDeletionTracking), + _settings.DisableDeletionTracking ? Parameters.Empty() : Parameters.DeletedStreamId, + _settings.DisableDeletionTracking ? Parameters.Empty() : Parameters.DeletedStreamIdOriginal, + Parameters.CreatedUtc(_settings.GetUtcNow?.Invoke()), + _settings.DisableDeletionTracking + ? Parameters.Empty() + : Parameters.DeletedMessages(streamIdInfo.PostgresqlStreamId, eventIds))) + { + await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + } + } +} diff --git a/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.ListStreams.cs b/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.ListStreams.cs new file mode 100644 index 000000000..25a8cd4b1 --- /dev/null +++ b/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.ListStreams.cs @@ -0,0 +1,77 @@ +namespace SqlStreamStore +{ + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + using Npgsql; + using SqlStreamStore.PgSqlScriptsV2; + using SqlStreamStore.Streams; + + public partial class PostgresStreamStoreV2 + { + protected override async Task ListStreamsInternal( + Pattern pattern, + int maxCount, + string continuationToken, + ListNextStreamsPage listNextStreamsPage, + CancellationToken cancellationToken) + { + if(!int.TryParse(continuationToken, out var afterIdInternal)) + { + afterIdInternal = -1; + } + + var streamIds = new List(); + + using(var connection = await OpenConnection(cancellationToken)) + using(var transaction = connection.BeginTransaction()) + using(var command = GetListStreamsCommand(pattern, maxCount, afterIdInternal, transaction)) + using(var reader = await command + .ExecuteReaderAsync(cancellationToken) + .ConfigureAwait(false)) + { + while(await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + streamIds.Add(reader.GetString(0)); + afterIdInternal = reader.GetInt32(1); + } + } + + return new ListStreamsPage(afterIdInternal.ToString(), streamIds.ToArray(), listNextStreamsPage); + } + + private NpgsqlCommand GetListStreamsCommand( + Pattern pattern, + int maxCount, + int? afterIdInternal, + NpgsqlTransaction transaction) + { + switch(pattern) + { + case Pattern.Any _: + return BuildFunctionCommand( + _schema.ListStreams, + transaction, + Parameters.MaxCount(maxCount), + Parameters.OptionalAfterIdInternal(afterIdInternal)); + case Pattern.StartingWith p: + return BuildFunctionCommand( + _schema.ListStreamsStartingWith, + transaction, + Parameters.Pattern(pattern.Value), + Parameters.MaxCount(maxCount), + Parameters.OptionalAfterIdInternal(afterIdInternal)); + case Pattern.EndingWith p: + return BuildFunctionCommand( + _schema.ListStreamsEndingWith, + transaction, + Parameters.Pattern(pattern.Value), + Parameters.MaxCount(maxCount), + Parameters.OptionalAfterIdInternal(afterIdInternal)); + + default: + throw Pattern.Unrecognized(nameof(pattern)); + } + } + } +} diff --git a/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Metadata.cs b/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Metadata.cs new file mode 100644 index 000000000..c0ebba718 --- /dev/null +++ b/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Metadata.cs @@ -0,0 +1,101 @@ +namespace SqlStreamStore +{ + using System.Threading; + using System.Threading.Tasks; + using Npgsql; + using SqlStreamStore.PgSqlScriptsV2; + using SqlStreamStore.Streams; + + partial class PostgresStreamStoreV2 + { + protected override async Task GetStreamMetadataInternal( + string streamId, + CancellationToken cancellationToken) + { + var streamIdInfo = new StreamIdInfo(streamId); + + using(var connection = await OpenConnection(cancellationToken)) + using(var transaction = connection.BeginTransaction()) + { + return await GetStreamMetadataInternal(streamIdInfo, transaction, cancellationToken); + } + } + + private async Task GetStreamMetadataInternal( + StreamIdInfo streamIdInfo, + NpgsqlTransaction transaction, + CancellationToken cancellationToken) + { + var page = await ReadStreamInternal( + streamIdInfo.MetadataPosgresqlStreamId, + StreamVersion.End, + 1, + ReadDirection.Backward, + true, + null, + transaction, + cancellationToken); + + if(page.Status == PageReadStatus.StreamNotFound) + { + return new StreamMetadataResult(streamIdInfo.PostgresqlStreamId.IdOriginal, -1); + } + + var metadataMessage = await page.Messages[0].GetJsonDataAs(cancellationToken); + + return new StreamMetadataResult( + streamIdInfo.PostgresqlStreamId.IdOriginal, + page.LastStreamVersion, + metadataMessage.MaxAge, + metadataMessage.MaxCount, + metadataMessage.MetaJson); + } + + protected override async Task SetStreamMetadataInternal( + string streamId, + int expectedStreamMetadataVersion, + int? maxAge, + int? maxCount, + string metadataJson, + CancellationToken cancellationToken) + { + int currentVersion; + + var metadata = new MetadataMessage + { + StreamId = streamId, + MaxAge = maxAge, + MaxCount = maxCount, + MetaJson = metadataJson + }; + + var streamIdInfo = new StreamIdInfo(streamId); + + using(var connection = await OpenConnection(cancellationToken)) + using(var transaction = connection.BeginTransaction()) + using(var command = BuildFunctionCommand( + _schema.SetStreamMetadata, + transaction, + Parameters.StreamId(streamIdInfo.PostgresqlStreamId), + Parameters.MetadataStreamId(streamIdInfo.MetadataPosgresqlStreamId), + Parameters.MetadataStreamIdOriginal(streamIdInfo.MetadataPosgresqlStreamId), + Parameters.OptionalMaxAge(metadata.MaxAge), + Parameters.OptionalMaxCount(metadata.MaxCount), + Parameters.ExpectedVersion(expectedStreamMetadataVersion), + Parameters.CreatedUtc(_settings.GetUtcNow?.Invoke()), + Parameters.MetadataStreamMessage( + streamIdInfo.MetadataPosgresqlStreamId, + expectedStreamMetadataVersion, + metadata))) + { + currentVersion = (int) await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false); + + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + } + + await TryScavenge(streamIdInfo, cancellationToken); + + return new SetStreamMetadataResult(currentVersion); + } + } +} diff --git a/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Read.cs b/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Read.cs new file mode 100644 index 000000000..fe2aa15f8 --- /dev/null +++ b/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Read.cs @@ -0,0 +1,278 @@ +namespace SqlStreamStore +{ + using System; + using System.Collections.Generic; + using System.Data; + using System.Data.Common; + using System.Linq; + using System.Text; + using System.Threading; + using System.Threading.Tasks; + using Npgsql; + using SqlStreamStore.PgSqlScriptsV2; + using SqlStreamStore.Streams; + + partial class PostgresStreamStoreV2 + { + private async Task ReadStreamInternal( + PostgresqlStreamId streamId, + int start, + int count, + ReadDirection direction, + bool prefetch, + ReadNextStreamPage readNext, + NpgsqlTransaction transaction, + CancellationToken cancellationToken) + { + // If the count is int.MaxValue, TSql will see it as a negative number. + // Users shouldn't be using int.MaxValue in the first place anyway. + count = count == int.MaxValue ? count - 1 : count; + + // To read backwards from end, need to use int MaxValue + var streamVersion = start == StreamVersion.End ? int.MaxValue : start; + + var messages = new List<(StreamMessage message, int? maxAge)>(); + + Func, int, int> getNextVersion; + + if(direction == ReadDirection.Forward) + { + getNextVersion = (events, lastVersion) => + { + if(events.Any()) + { + return events.Last().StreamVersion + 1; + } + + return lastVersion + 1; + }; + } + else + { + getNextVersion = (events, lastVersion) => + { + if(events.Any()) + { + return events.Last().StreamVersion - 1; + } + + return -1; + }; + } + + var refcursorSql = new StringBuilder(); + + using(var command = BuildFunctionCommand( + _schema.Read, + transaction, + Parameters.StreamId(streamId), + Parameters.Count(count + 1), + Parameters.Version(streamVersion), + Parameters.ReadDirection(direction), + Parameters.Prefetch(prefetch))) + using(var reader = await command + .ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken) + .ConfigureAwait(false)) + { + while(await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + refcursorSql.AppendLine(Schema.FetchAll(reader.GetString(0))); + } + } + + using(var command = new NpgsqlCommand(refcursorSql.ToString(), transaction.Connection, transaction)) + using(var reader = await command + .ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken) + .ConfigureAwait(false)) + { + if(!reader.HasRows) + { + return new ReadStreamPage( + streamId.IdOriginal, + PageReadStatus.StreamNotFound, + start, + -1, + -1, + -1, + direction, + true, + readNext); + } + + if(messages.Count == count) + { + messages.Add(default); + } + + await reader.ReadAsync(cancellationToken).ConfigureAwait(false); + + var lastVersion = reader.GetInt32(0); + var lastPosition = reader.GetInt64(1); + var maxAge = reader.GetFieldValue(2); + + await reader.NextResultAsync(cancellationToken).ConfigureAwait(false); + + while(await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + messages.Add((await ReadStreamMessage(reader, streamId, prefetch), maxAge)); + } + + var isEnd = true; + + if(messages.Count == count + 1) + { + isEnd = false; + messages.RemoveAt(count); + } + + var filteredMessages = FilterExpired(messages); + + return new ReadStreamPage( + streamId.IdOriginal, + PageReadStatus.Success, + start, + getNextVersion(filteredMessages, lastVersion), + lastVersion, + lastPosition, + direction, + isEnd, + readNext, + filteredMessages.ToArray()); + } + } + + protected override async Task ReadStreamForwardsInternal( + string streamId, + int start, + int count, + bool prefetch, + ReadNextStreamPage readNext, + CancellationToken cancellationToken) + { + var streamIdInfo = new StreamIdInfo(streamId); + + using(var connection = await OpenConnection(cancellationToken)) + using(var transaction = connection.BeginTransaction()) + { + return await ReadStreamInternal(streamIdInfo.PostgresqlStreamId, + start, + count, + ReadDirection.Forward, + prefetch, + readNext, + transaction, + cancellationToken); + } + } + + protected override async Task ReadStreamBackwardsInternal( + string streamId, + int fromVersionInclusive, + int count, + bool prefetch, + ReadNextStreamPage readNext, + CancellationToken cancellationToken) + { + var streamIdInfo = new StreamIdInfo(streamId); + + using(var connection = await OpenConnection(cancellationToken)) + using(var transaction = connection.BeginTransaction()) + { + return await ReadStreamInternal(streamIdInfo.PostgresqlStreamId, + fromVersionInclusive, + count, + ReadDirection.Backward, + prefetch, + readNext, + transaction, + cancellationToken); + } + } + + private async Task ReadStreamMessage( + DbDataReader reader, + PostgresqlStreamId streamId, + bool prefetch) + { + async Task ReadString(int ordinal) + { + if(reader.IsDBNull(ordinal)) + { + return null; + } + + using(var textReader = reader.GetTextReader(ordinal)) + { + return await textReader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var messageId = reader.GetGuid(1); + var streamVersion = reader.GetInt32(2); + var position = reader.GetInt64(3); + var createdUtc = reader.GetDateTime(4); + var type = reader.GetString(5); + var jsonMetadata = await ReadString(6); + + if(prefetch) + { + return new StreamMessage( + streamId.IdOriginal, + messageId, + streamVersion, + position, + createdUtc, + type, + jsonMetadata, + await ReadString(7)); + } + + return + new StreamMessage( + streamId.IdOriginal, + messageId, + streamVersion, + position, + createdUtc, + type, + jsonMetadata, + ct => GetJsonData(streamId, streamVersion)(ct)); + } + + protected override async Task ReadHeadPositionInternal(CancellationToken cancellationToken) + { + using(var connection = await OpenConnection(cancellationToken)) + using(var transaction = connection.BeginTransaction()) + using(var command = BuildFunctionCommand(_schema.ReadAllHeadPosition, transaction)) + { + var result = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false); + + return result == DBNull.Value ? Position.End : (long) result; + } + } + + protected override async Task ReadStreamHeadPositionInternal(string streamId, CancellationToken cancellationToken) + { + using(var connection = await OpenConnection(cancellationToken)) + using(var transaction = connection.BeginTransaction()) + using(var command = BuildFunctionCommand(_schema.ReadStreamHeadPosition, transaction, Parameters.StreamId(new PostgresqlStreamId(streamId)))) + { + var result = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false); + + return result == DBNull.Value ? Position.End : (long) result; + } + } + + protected override async Task ReadStreamHeadVersionInternal(string streamId, CancellationToken cancellationToken) + { + using(var connection = await OpenConnection(cancellationToken)) + using(var transaction = connection.BeginTransaction()) + using(var command = BuildFunctionCommand(_schema.ReadStreamHeadVersion, transaction, Parameters.StreamId(new PostgresqlStreamId(streamId)))) + { + var result = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false); + + return result == DBNull.Value ? StreamVersion.End : (int) result; + } + } + } +} diff --git a/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.ReadAll.cs b/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.ReadAll.cs new file mode 100644 index 000000000..c3f081fda --- /dev/null +++ b/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.ReadAll.cs @@ -0,0 +1,212 @@ +namespace SqlStreamStore +{ + using System; + using System.Collections.Generic; + using System.Data; + using System.Data.Common; + using System.Threading; + using System.Threading.Tasks; + using SqlStreamStore.PgSqlScriptsV2; + using SqlStreamStore.Streams; + + partial class PostgresStreamStoreV2 + { + protected override async Task ReadAllForwardsInternal( + long fromPositionExclusive, + int maxCount, + bool prefetch, + ReadNextAllPage readNext, + CancellationToken cancellationToken) + { + maxCount = maxCount == int.MaxValue ? maxCount - 1 : maxCount; + + using(var connection = await OpenConnection(cancellationToken)) + using(var transaction = connection.BeginTransaction()) + using(var command = BuildFunctionCommand( + _schema.ReadAll, + transaction, + Parameters.Count(maxCount + 1), + Parameters.Position(fromPositionExclusive), + Parameters.ReadDirection(ReadDirection.Forward), + Parameters.Prefetch(prefetch))) + using(var reader = await command + .ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken) + .ConfigureAwait(false)) + { + if(!reader.HasRows) + { + return new ReadAllPage( + fromPositionExclusive, + fromPositionExclusive, + true, + ReadDirection.Forward, + readNext, + Array.Empty()); + } + + var messages = new List<(StreamMessage message, int? maxAge)>(); + + while(await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + if(messages.Count == maxCount) + { + messages.Add(default); + } + else + { + var streamIdInfo = new StreamIdInfo(reader.GetString(0)); + var (message, maxAge, _) = + await ReadAllStreamMessage(reader, streamIdInfo.PostgresqlStreamId, prefetch); + messages.Add((message, maxAge)); + } + } + + bool isEnd = true; + + if(messages.Count == maxCount + 1) // An extra row was read, we're not at the end + { + isEnd = false; + messages.RemoveAt(maxCount); + } + + var filteredMessages = FilterExpired(messages); + + var nextPosition = filteredMessages[filteredMessages.Count - 1].Position + 1; + + return new ReadAllPage( + fromPositionExclusive, + nextPosition, + isEnd, + ReadDirection.Forward, + readNext, + filteredMessages.ToArray()); + } + } + + protected override async Task ReadAllBackwardsInternal( + long fromPositionExclusive, + int maxCount, + bool prefetch, + ReadNextAllPage readNext, + CancellationToken cancellationToken) + { + maxCount = maxCount == int.MaxValue ? maxCount - 1 : maxCount; + var ordinal = fromPositionExclusive == Position.End ? long.MaxValue : fromPositionExclusive; + + using(var connection = await OpenConnection(cancellationToken)) + using(var transaction = connection.BeginTransaction()) + using(var command = BuildFunctionCommand( + _schema.ReadAll, + transaction, + Parameters.Count(maxCount + 1), + Parameters.Position(ordinal), + Parameters.ReadDirection(ReadDirection.Backward), + Parameters.Prefetch(prefetch))) + using(var reader = await command + .ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken) + .ConfigureAwait(false)) + { + if(!reader.HasRows) + { + // When reading backwards and there are no more items, then next position is LongPosition.Start, + // regardless of what the fromPosition is. + return new ReadAllPage( + Position.Start, + Position.Start, + true, + ReadDirection.Backward, + readNext, + Array.Empty()); + } + + var messages = new List<(StreamMessage message, int? maxAge)>(); + + long lastOrdinal = 0; + while(await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + var streamIdInfo = new StreamIdInfo(reader.GetString(0)); + var (message, maxAge, position) = await ReadAllStreamMessage(reader, streamIdInfo.PostgresqlStreamId, prefetch); + messages.Add((message, maxAge)); + + lastOrdinal = position; + } + + bool isEnd = true; + var nextPosition = lastOrdinal; + + if(messages.Count == maxCount + 1) // An extra row was read, we're not at the end + { + isEnd = false; + messages.RemoveAt(maxCount); + } + + var filteredMessages = FilterExpired(messages); + + fromPositionExclusive = filteredMessages.Count > 0 ? filteredMessages[0].Position : 0; + + return new ReadAllPage( + fromPositionExclusive, + nextPosition, + isEnd, + ReadDirection.Backward, + readNext, + filteredMessages.ToArray()); + } + } + + private async Task<(StreamMessage message, int? maxAge, long position)> ReadAllStreamMessage( + DbDataReader reader, + PostgresqlStreamId streamId, + bool prefetch) + { + async Task ReadString(int ordinal) + { + if(reader.IsDBNull(ordinal)) + { + return null; + } + + using(var textReader = reader.GetTextReader(ordinal)) + { + return await textReader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var messageId = reader.GetGuid(1); + var streamVersion = reader.GetInt32(2); + var position = reader.GetInt64(3); + var createdUtc = reader.GetDateTime(4); + var type = reader.GetString(5); + var jsonMetadata = await ReadString(6); + + if(prefetch) + { + return ( + new StreamMessage( + streamId.IdOriginal, + messageId, + streamVersion, + position, + createdUtc, + type, + jsonMetadata, + await ReadString(7)), + reader.GetFieldValue(8), + position); + } + + return ( + new StreamMessage( + streamId.IdOriginal, + messageId, + streamVersion, + position, + createdUtc, + type, + jsonMetadata, + ct => GetJsonData(streamId, streamVersion)(ct)), + reader.GetFieldValue(8), + position); + } + } +} diff --git a/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Subscriptions.cs b/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Subscriptions.cs new file mode 100644 index 000000000..854d6aa27 --- /dev/null +++ b/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Subscriptions.cs @@ -0,0 +1,47 @@ +namespace SqlStreamStore +{ + using System; + using SqlStreamStore.Infrastructure; + using SqlStreamStore.Subscriptions; + + public partial class PostgresStreamStoreV2 + { + protected override IStreamSubscription SubscribeToStreamInternal( + string streamId, + int? startVersion, + StreamMessageReceived streamMessageReceived, + SubscriptionDropped subscriptionDropped, + HasCaughtUp hasCaughtUp, + bool prefetchJsonData, + string name) => new StreamSubscription( + streamId, + startVersion, + this, + GetStoreObservable, + streamMessageReceived, + subscriptionDropped, + hasCaughtUp, + prefetchJsonData, + name); + + + protected override IAllStreamSubscription SubscribeToAllInternal( + long? fromPosition, + AllStreamMessageReceived streamMessageReceived, + AllSubscriptionDropped subscriptionDropped, + HasCaughtUp hasCaughtUp, + bool prefetchJsonData, + string name) + => new AllStreamSubscription( + fromPosition, + this, + GetStoreObservable, + streamMessageReceived, + subscriptionDropped, + hasCaughtUp, + prefetchJsonData, + name); + + private IObservable GetStoreObservable => _streamStoreNotifier.Value; + } +} \ No newline at end of file diff --git a/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.cs b/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.cs new file mode 100644 index 000000000..d0a3be19a --- /dev/null +++ b/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.cs @@ -0,0 +1,272 @@ +namespace SqlStreamStore +{ + using System; + using System.Collections.Generic; + using System.Data; + using System.Threading; + using System.Threading.Tasks; + using Npgsql; + using SqlStreamStore.Infrastructure; + using SqlStreamStore.Logging; + using SqlStreamStore.PgSqlScriptsV2; + using SqlStreamStore.Subscriptions; + + /// + /// Represents a PostgreSQL stream store implementation. + /// + public partial class PostgresStreamStoreV2 : StreamStoreBase + { + private readonly PostgresStreamStoreV2Settings _settings; + private readonly Func _createConnection; + private readonly Schema _schema; + private readonly Lazy _streamStoreNotifier; + + public const int CurrentVersion = 2; + + /// + /// Initializes a new instance of + /// + /// A settings class to configure this instance. + public PostgresStreamStoreV2(PostgresStreamStoreV2Settings settings) + : base(settings.GetUtcNow, settings.LogName) + { + _settings = settings; + _createConnection = () => _settings.ConnectionFactory(_settings.ConnectionString); + _streamStoreNotifier = new Lazy(() => + { + if(_settings.CreateStreamStoreNotifier == null) + { + throw new InvalidOperationException( + "Cannot create notifier because supplied createStreamStoreNotifier was null"); + } + + return settings.CreateStreamStoreNotifier.Invoke(this); + }); + _schema = new Schema(_settings.Schema); + } + + private async Task OpenConnection(CancellationToken cancellationToken) + { + var connection = _createConnection(); + + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + + connection.ReloadTypes(); + + connection.TypeMapper.MapComposite(_schema.NewStreamMessage); + + if(_settings.ExplainAnalyze) + { + using(var command = new NpgsqlCommand(_schema.EnableExplainAnalyze, connection)) + { + await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + } + + return connection; + } + + /// + /// Creates a scheme that will hold streams and messages, if the schema does not exist. + /// Calls to this should part of an application's deployment/upgrade process and + /// not every time your application boots up. + /// + /// The cancellation instruction. + /// A representing the asynchronous operation. + public async Task CreateSchemaIfNotExists(CancellationToken cancellationToken = default) + { + using(var connection = _createConnection()) + { + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + using(var transaction = connection.BeginTransaction()) + { + using(var command = BuildCommand($"CREATE SCHEMA IF NOT EXISTS {_settings.Schema}", transaction)) + { + await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + using(var command = BuildCommand(_schema.Definition, transaction)) + { + await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + } + } + } + + /// + /// Drops all tables related to this store instance. + /// + /// The cancellation instruction. + /// A representing the asynchronous operation. + public async Task DropAll(CancellationToken cancellationToken = default) + { + GuardAgainstDisposed(); + + using(var connection = _createConnection()) + { + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + using(var transaction = connection.BeginTransaction()) + using(var command = BuildCommand(_schema.DropAll, transaction)) + { + await command + .ExecuteNonQueryAsync(cancellationToken) + .ConfigureAwait(false); + + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + } + } + } + + /// + /// Checks the store schema for the correct version. + /// + /// + /// A representing the result of the operation. + public async Task CheckSchema(CancellationToken cancellationToken = default) + { + using(var connection = _createConnection()) + { + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + using(var transaction = connection.BeginTransaction()) + using(var command = BuildFunctionCommand(_schema.ReadSchemaVersion, transaction)) + { + var result = (int) await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false); + + return new CheckSchemaResult(result, CurrentVersion); + } + } + } + + private Func> GetJsonData(PostgresqlStreamId streamId, int version) + => async cancellationToken => + { + using(var connection = await OpenConnection(cancellationToken)) + using(var transaction = connection.BeginTransaction()) + using(var command = BuildFunctionCommand( + _schema.ReadJsonData, + transaction, + Parameters.StreamId(streamId), + Parameters.Version(version))) + using(var reader = await command + .ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken) + .ConfigureAwait(false)) + { + if(!await reader.ReadAsync(cancellationToken).ConfigureAwait(false) || reader.IsDBNull(0)) + { + return null; + } + + using(var textReader = reader.GetTextReader(0)) + { + return await textReader.ReadToEndAsync().ConfigureAwait(false); + } + } + }; + + private static NpgsqlCommand BuildFunctionCommand( + string function, + NpgsqlTransaction transaction, + params NpgsqlParameter[] parameters) + { + var command = new NpgsqlCommand(function, transaction.Connection, transaction) + { + CommandType = CommandType.StoredProcedure, + }; + + foreach(var parameter in parameters) + { + command.Parameters.Add(parameter); + } + + return command; + } + + private static NpgsqlCommand BuildCommand( + string commandText, + NpgsqlTransaction transaction) => new NpgsqlCommand(commandText, transaction.Connection, transaction); + + internal async Task TryScavenge( + StreamIdInfo streamIdInfo, + CancellationToken cancellationToken) + { + if(streamIdInfo.PostgresqlStreamId == PostgresqlStreamId.Deleted) + { + return -1; + } + + try + { + using(var connection = await OpenConnection(cancellationToken)) + using(var transaction = connection.BeginTransaction()) + { + var deletedMessageIds = new List(); + using(var command = BuildFunctionCommand( + _schema.Scavenge, + transaction, + Parameters.StreamId(streamIdInfo.PostgresqlStreamId))) + using(var reader = await command + .ExecuteReaderAsync(cancellationToken) + .ConfigureAwait(false)) + { + while(await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + deletedMessageIds.Add(reader.GetGuid(0)); + } + } + + Logger.Info( + "Found {count} message(s) for stream {streamId} to scavenge.", + deletedMessageIds.Count, + streamIdInfo.PostgresqlStreamId); + + if(deletedMessageIds.Count > 0) + { + Logger.Debug( + "Scavenging the following messages on stream {streamId}: {messageIds}", + streamIdInfo.PostgresqlStreamId, + deletedMessageIds); + + await DeleteEventsInternal( + streamIdInfo, + deletedMessageIds.ToArray(), + transaction, + cancellationToken).ConfigureAwait(false); + } + + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + + return deletedMessageIds.Count; + } + } + catch(Exception ex) + { + Logger.WarnException( + "Scavenge attempt failed on stream {streamId}. Another attempt will be made when this stream is written to.", + ex, + streamIdInfo.PostgresqlStreamId.IdOriginal); + } + + return -1; + } + + /// + /// Returns the script that can be used to create the Sql Stream Store in a Postgres database. + /// + /// The database creation script. + public string GetSchemaCreationScript() + { + return _schema.Definition; + } + + /// + /// Returns the script that can be used to migrate to the latest schema version 2. + /// + /// The database creation script. + public string GetMigrationScript() + { + return _schema.Migration; + } + } +} diff --git a/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2Settings.cs b/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2Settings.cs new file mode 100644 index 000000000..caea979da --- /dev/null +++ b/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2Settings.cs @@ -0,0 +1,100 @@ +namespace SqlStreamStore +{ + using System; + using Npgsql; + using SqlStreamStore.Imports.Ensure.That; + using SqlStreamStore.Infrastructure; + using SqlStreamStore.Subscriptions; + + public class PostgresStreamStoreV2Settings + { + private string _schema = "public"; + private Func _connectionFactory; + private GetUtcNow _getUtcNow = SystemClock.GetUtcNow; + + /// + /// Initializes a new instance of . + /// + /// The connection string. + public PostgresStreamStoreV2Settings(string connectionString) + { + Ensure.That(connectionString, nameof(connectionString)).IsNotNullOrWhiteSpace(); + + ConnectionString = connectionString; + } + + /// + /// Gets the connection string. + /// + public string ConnectionString { get; } + + /// + /// Allows overriding of the stream store notifier. The default implementation + /// creates . + /// + public CreateStreamStoreNotifier CreateStreamStoreNotifier { get; set; } = + store => new PollingStreamStoreNotifier(store); + + /// + /// The schema SQL Stream Store should place database objects into. Defaults to "public". + /// + public string Schema + { + get => _schema; + set + { + Ensure.That(value, nameof(Schema)).IsNotNullOrWhiteSpace(); + + _schema = value; + } + } + + /// + /// Loads the auto_explain module and turns it on for all queries. Useful for index tuning. + /// + public bool ExplainAnalyze { get; set; } + + /// + /// A delegate to return the current UTC now. Used in testing to + /// control timestamps and time related operations. + /// + public GetUtcNow GetUtcNow + { + get => _getUtcNow; + set => _getUtcNow = value ?? SystemClock.GetUtcNow; + } + + /// + /// The log name used for any of the log messages. + /// + public string LogName { get; } = nameof(PostgresStreamStoreV2); + + /// + /// Allows setting whether or not deleting expired (i.e., older than maxCount) messages happens in the same database transaction as append to stream or not. + /// This does not effect scavenging when setting a stream's metadata - it is always run in the same transaction. + /// + public bool ScavengeAsynchronously { get; set; } = true; + + /// + /// Allows overriding the way a is created given a connection string. + /// The default implementation simply passes the connection string into the constructor. + /// + public Func ConnectionFactory + { + get => _connectionFactory + ?? (_connectionFactory = connectionString => new NpgsqlConnection(connectionString)); + set + { + Ensure.That(value, nameof(value)).IsNotNull(); + _connectionFactory = value; + } + } + + /// + /// Disables stream and message deletion tracking. Will increase + /// performance, however subscribers won't know if a stream or a + /// message has been deleted. This can be modified at runtime. + /// + public bool DisableDeletionTracking { get; set; } + } +} \ No newline at end of file diff --git a/src/SqlStreamStore.PostgresV2/Properties/AssemblyInfo.cs b/src/SqlStreamStore.PostgresV2/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..fbcd872de --- /dev/null +++ b/src/SqlStreamStore.PostgresV2/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("SqlStreamStore.Postgres.V2.Tests")] \ No newline at end of file diff --git a/src/SqlStreamStore.PostgresV2/SqlStreamStore.PostgresV2.csproj b/src/SqlStreamStore.PostgresV2/SqlStreamStore.PostgresV2.csproj new file mode 100644 index 000000000..9f551ec94 --- /dev/null +++ b/src/SqlStreamStore.PostgresV2/SqlStreamStore.PostgresV2.csproj @@ -0,0 +1,20 @@ + + + Postgre SQL provider for SQL StreamStore + Stream Store - Postgres + netstandard2.0 + SqlStreamStore.PostgresV2 + SqlStreamStore.PostgresV2 + SqlStreamStore + + + + + + + + + + + + \ No newline at end of file diff --git a/src/SqlStreamStore.SchemaCreationScriptTool/SqlStreamStore.SchemaCreationScriptTool.csproj b/src/SqlStreamStore.SchemaCreationScriptTool/SqlStreamStore.SchemaCreationScriptTool.csproj index 3381c6f9f..d284c7bba 100644 --- a/src/SqlStreamStore.SchemaCreationScriptTool/SqlStreamStore.SchemaCreationScriptTool.csproj +++ b/src/SqlStreamStore.SchemaCreationScriptTool/SqlStreamStore.SchemaCreationScriptTool.csproj @@ -4,7 +4,7 @@ A tool to generate the schema creation script of the various SQL based implementations. SQL Stream Store - Schema Creation Script Tool Exe - netcoreapp2.1 + net6.0 true SqlStreamStore.SchemaCreationScriptTool SqlStreamStore.SchemaCreationScriptTool @@ -23,9 +23,9 @@ - - - - + + + + \ No newline at end of file diff --git a/src/SqlStreamStore.Sqlite/SqlStreamStore.Sqlite.csproj b/src/SqlStreamStore.Sqlite/SqlStreamStore.Sqlite.csproj index f25a980ec..1c1d4dc07 100644 --- a/src/SqlStreamStore.Sqlite/SqlStreamStore.Sqlite.csproj +++ b/src/SqlStreamStore.Sqlite/SqlStreamStore.Sqlite.csproj @@ -15,8 +15,8 @@ - - + + diff --git a/src/SqlStreamStore.TestUtils/MsSql/SqlServerContainer.cs b/src/SqlStreamStore.TestUtils/MsSql/SqlServerContainer.cs index 4eeed5339..d5796e36e 100644 --- a/src/SqlStreamStore.TestUtils/MsSql/SqlServerContainer.cs +++ b/src/SqlStreamStore.TestUtils/MsSql/SqlServerContainer.cs @@ -37,7 +37,7 @@ public SqlConnection CreateConnection() => new SqlConnection(CreateConnectionStringBuilder().ConnectionString); public SqlConnectionStringBuilder CreateConnectionStringBuilder() - => new SqlConnectionStringBuilder($"server=localhost,{HostPort};User Id=sa;Password={Password};Initial Catalog=master"); + => new SqlConnectionStringBuilder($"server=localhost,{HostPort};User Id=sa;Password={Password};Initial Catalog=master;TrustServerCertificate=true"); public async Task Start(CancellationToken cancellationToken = default) { diff --git a/src/SqlStreamStore.TestUtils/SqlStreamStore.TestUtils.csproj b/src/SqlStreamStore.TestUtils/SqlStreamStore.TestUtils.csproj index 4c7a7731b..76afc7b59 100644 --- a/src/SqlStreamStore.TestUtils/SqlStreamStore.TestUtils.csproj +++ b/src/SqlStreamStore.TestUtils/SqlStreamStore.TestUtils.csproj @@ -1,21 +1,21 @@  - netcoreapp2.1 + net6.0 SqlStreamStore.TestUtils false - - - - - - - - + + + + + + + + - - + + all diff --git a/src/SqlStreamStore/SqlStreamStore.csproj b/src/SqlStreamStore/SqlStreamStore.csproj index 56b7e7087..2cc09fa01 100644 --- a/src/SqlStreamStore/SqlStreamStore.csproj +++ b/src/SqlStreamStore/SqlStreamStore.csproj @@ -9,9 +9,6 @@ SqlStreamStore - - - @@ -19,4 +16,5 @@ runtime; build; native; contentfiles; analyzers + \ No newline at end of file diff --git a/tests/Directory.Build.props b/tests/Directory.Build.props index 1c0923a44..2029261a7 100644 --- a/tests/Directory.Build.props +++ b/tests/Directory.Build.props @@ -7,8 +7,8 @@ - - + + all diff --git a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.StreamLimits.cs b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.StreamLimits.cs index 4f70de6e6..03663e64e 100644 --- a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.StreamLimits.cs +++ b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.StreamLimits.cs @@ -63,7 +63,7 @@ await Store [Fact, Trait("Category", "StreamMetadata")] public async Task When_stream_has_expired_messages_and_read_forwards_then_should_not_get_expired_messages() { - var currentUtc = new DateTime(2016, 1, 1, 0, 0, 0); + var currentUtc = new DateTime(2016, 1, 1, 0, 0, 0, DateTimeKind.Utc); Fixture.GetUtcNow = () => currentUtc; string streamId = "stream-1"; await Store @@ -82,7 +82,7 @@ await Store [Fact, Trait("Category", "StreamMetadata")] public async Task When_stream_has_expired_messages_and_read_backward_then_should_not_get_expired_messages() { - var currentUtc = new DateTime(2016, 1, 1, 0, 0, 0); + var currentUtc = new DateTime(2016, 1, 1, 0, 0, 0, DateTimeKind.Utc); Fixture.GetUtcNow = () => currentUtc; const string streamId = "stream-1"; await Store @@ -101,7 +101,7 @@ await Store [Fact, Trait("Category", "StreamMetadata")] public async Task When_streams_have_expired_messages_and_read_all_forwards_then_should_not_get_expired_messages() { - var currentUtc = new DateTime(2016, 1, 1, 0, 0, 0); + var currentUtc = new DateTime(2016, 1, 1, 0, 0, 0, DateTimeKind.Utc); Fixture.GetUtcNow = () => currentUtc; // Arrange const string streamId1 = "stream-1"; @@ -134,7 +134,7 @@ await Store [Fact, Trait("Category", "StreamMetadata")] public async Task When_streams_have_expired_messages_and_read_all_backwards_then_should_not_get_expired_messages() { - var currentUtc = new DateTime(2016, 1, 1, 0, 0, 0); + var currentUtc = new DateTime(2016, 1, 1, 0, 0, 0, DateTimeKind.Utc); Fixture.GetUtcNow = () => currentUtc; // Arrange diff --git a/tests/SqlStreamStore.HAL.Tests/SqlStreamStore.HAL.Tests.csproj b/tests/SqlStreamStore.HAL.Tests/SqlStreamStore.HAL.Tests.csproj index 71272c22f..bc504238d 100644 --- a/tests/SqlStreamStore.HAL.Tests/SqlStreamStore.HAL.Tests.csproj +++ b/tests/SqlStreamStore.HAL.Tests/SqlStreamStore.HAL.Tests.csproj @@ -1,11 +1,11 @@  - netcoreapp2.1 + net6.0 SqlStreamStore.HAL.Tests SqlStreamStore.HAL.Tests - + diff --git a/tests/SqlStreamStore.Http.Tests/SqlStreamStore.Http.Tests.csproj b/tests/SqlStreamStore.Http.Tests/SqlStreamStore.Http.Tests.csproj index 6ebc9faf6..e7b5343c9 100644 --- a/tests/SqlStreamStore.Http.Tests/SqlStreamStore.Http.Tests.csproj +++ b/tests/SqlStreamStore.Http.Tests/SqlStreamStore.Http.Tests.csproj @@ -1,7 +1,7 @@  - netcoreapp3.1 + net6.0 SqlStreamStore.Http.Tests SqlStreamStore.Http.Tests SqlStreamStore @@ -19,6 +19,6 @@ - + diff --git a/tests/SqlStreamStore.MsSql.Tests/SqlStreamStore.MsSql.Tests.csproj b/tests/SqlStreamStore.MsSql.Tests/SqlStreamStore.MsSql.Tests.csproj index a7a500e40..236973360 100644 --- a/tests/SqlStreamStore.MsSql.Tests/SqlStreamStore.MsSql.Tests.csproj +++ b/tests/SqlStreamStore.MsSql.Tests/SqlStreamStore.MsSql.Tests.csproj @@ -1,6 +1,6 @@  - netcoreapp3.1 + net6.0 SqlStreamStore.MsSql.Tests SqlStreamStore.MsSql.Tests SqlStreamStore @@ -12,5 +12,4 @@ - \ No newline at end of file diff --git a/tests/SqlStreamStore.MsSql.V3.Tests/SqlStreamStore.MsSql.V3.Tests.csproj b/tests/SqlStreamStore.MsSql.V3.Tests/SqlStreamStore.MsSql.V3.Tests.csproj index 419e38c6e..234500363 100644 --- a/tests/SqlStreamStore.MsSql.V3.Tests/SqlStreamStore.MsSql.V3.Tests.csproj +++ b/tests/SqlStreamStore.MsSql.V3.Tests/SqlStreamStore.MsSql.V3.Tests.csproj @@ -1,6 +1,6 @@  - netcoreapp3.1 + net6.0 SqlStreamStore.MsSql.V3.Tests SqlStreamStore @@ -15,5 +15,4 @@ - \ No newline at end of file diff --git a/tests/SqlStreamStore.MySql.Tests/SqlStreamStore.MySql.Tests.csproj b/tests/SqlStreamStore.MySql.Tests/SqlStreamStore.MySql.Tests.csproj index 00a35b777..657a35ece 100644 --- a/tests/SqlStreamStore.MySql.Tests/SqlStreamStore.MySql.Tests.csproj +++ b/tests/SqlStreamStore.MySql.Tests/SqlStreamStore.MySql.Tests.csproj @@ -1,6 +1,6 @@  - netcoreapp3.1 + net6.0 SqlStreamStore.MySql.Tests SqlStreamStore @@ -11,5 +11,4 @@ - diff --git a/tests/SqlStreamStore.Postgres.Tests/SqlStreamStore.Postgres.Tests.csproj b/tests/SqlStreamStore.Postgres.Tests/SqlStreamStore.Postgres.Tests.csproj index 23e02cc30..52a732a77 100644 --- a/tests/SqlStreamStore.Postgres.Tests/SqlStreamStore.Postgres.Tests.csproj +++ b/tests/SqlStreamStore.Postgres.Tests/SqlStreamStore.Postgres.Tests.csproj @@ -1,7 +1,7 @@  - netcoreapp3.1 + net6.0 SqlStreamStore.Postgres.Tests SqlStreamStore diff --git a/tests/SqlStreamStore.Postgres.V2.Tests/PostgresStreamStoreAcceptanceTests.cs b/tests/SqlStreamStore.Postgres.V2.Tests/PostgresStreamStoreAcceptanceTests.cs new file mode 100644 index 000000000..784bee6af --- /dev/null +++ b/tests/SqlStreamStore.Postgres.V2.Tests/PostgresStreamStoreAcceptanceTests.cs @@ -0,0 +1,157 @@ +namespace SqlStreamStore +{ + using System.Threading.Tasks; + using Shouldly; + using SqlStreamStore.PgSqlScriptsV2; + using Xunit; + using Xunit.Abstractions; + + public class PostgresStreamStoreAcceptanceTests : AcceptanceTests, IClassFixture + { + private readonly PostgresStreamStoreV3FixturePool _fixturePool; + + public PostgresStreamStoreAcceptanceTests( + PostgresStreamStoreV3FixturePool fixturePool, + ITestOutputHelper testOutputHelper) + : base(testOutputHelper) + { + _fixturePool = fixturePool; + } + + protected override async Task CreateFixture() + => await _fixturePool.Get(TestOutputHelper); + + [Fact] + public async Task Can_check_schema() + { + using(var fixture = await _fixturePool.Get(TestOutputHelper)) + { + var result = await fixture.PostgresStreamStore.CheckSchema(); + + result.ShouldBe(new CheckSchemaResult(2, 2)); + result.IsMatch.ShouldBeTrue(); + } + } + + [Fact] + public async Task Can_export_database_migration_script() + { + var schema = "custom_schema"; + using(var fixture = await _fixturePool.Get(TestOutputHelper, schema)) + { + var sqlScript = fixture.PostgresStreamStore.GetMigrationScript(); + sqlScript.ShouldBe(new Scripts(schema).Migration); + } + } + + /* + [Fact] + public async Task Can_use_multiple_schemas() + { + using (var dboFixture = await CreateFixture()) + { + var dboStore = dboFixture.Store; + + using(var barFixture = await CreateFixture()) + { + var barStore = barFixture.Store; + + await dboStore.AppendToStream("stream-1", + ExpectedVersion.NoStream, + CreateNewStreamMessages(1, 2)); + await barStore.AppendToStream("stream-1", + ExpectedVersion.NoStream, + CreateNewStreamMessages(1, 2)); + + var dboHeadPosition = await dboStore.ReadHeadPosition(); + var barHeadPosition = await barStore.ReadHeadPosition(); + + dboHeadPosition.ShouldBe(1); + barHeadPosition.ShouldBe(1); + } + } + } + + [Fact] + public async Task when_try_scavenge_fails_returns_negative_one() + { + using (var fixture = await PostgresStreamStoreFixture.Create(testOutputHelper: TestOutputHelper)) + { + var cts = new CancellationTokenSource(); + + cts.Cancel(); + + var result = await fixture.Store.TryScavenge(new StreamIdInfo("stream-1"), cts.Token); + + result.ShouldBe(-1); + } + } + + [Theory, InlineData("dbo"), InlineData("myschema")] + public async Task Can_call_initialize_repeatably(string schema) + { + using(var fixture = await PostgresStreamStoreFixture.Create(schema, testOutputHelper: TestOutputHelper, createSchema: false)) + { + await fixture.Store.CreateSchemaIfNotExists(); + await fixture.Store.CreateSchemaIfNotExists(); + } + } + + [Fact] + public async Task Can_drop_all() + { + var streamStoreObjects = new List(); + + string ReadInformationSchema((string name, string table) _) + => $"SELECT {_.name}_name FROM information_schema.{_.table} WHERE {_.name}_schema = 'dbo'"; + + using(var fixture = await PostgresStreamStoreFixture.Create(testOutputHelper: TestOutputHelper)) + { + await fixture.Store.DropAll(); + + var commandText = string.Join( + $"{Environment.NewLine}UNION{Environment.NewLine}", + new[] + { + ("table", "tables"), + ("sequence", "sequences"), + ("constraint", "table_constraints"), + ("user_defined_type", "user_defined_types"), + ("routine", "routines") + }.Select(ReadInformationSchema)); + + using(var connection = new NpgsqlConnection(fixture.ConnectionString)) + { + await connection.OpenAsync().ConfigureAwait(false); + + using(var command = new NpgsqlCommand(commandText, connection)) + using(var reader = await command.ExecuteReaderAsync().ConfigureAwait(false)) + { + while(await reader.ReadAsync().ConfigureAwait(false)) + { + streamStoreObjects.Add(reader.GetString(0)); + } + } + } + + streamStoreObjects.ShouldBeEmpty(); + } + } + + + [Fact] + public void Can_export_database_creation_script() + { + string schema = "custom_schema"; + var store = new PostgresStreamStore(new PostgresStreamStoreSettings("server=.;database=sss") + { + Schema = schema, + }); + + var sqlScript = store.GetSchemaCreationScript(); + sqlScript.ShouldBe(new Scripts(schema).CreateSchema); + } + */ + + } +} diff --git a/tests/SqlStreamStore.Postgres.V2.Tests/PostgresStreamStoreFixture.cs b/tests/SqlStreamStore.Postgres.V2.Tests/PostgresStreamStoreFixture.cs new file mode 100644 index 000000000..a1b8193e4 --- /dev/null +++ b/tests/SqlStreamStore.Postgres.V2.Tests/PostgresStreamStoreFixture.cs @@ -0,0 +1,103 @@ +namespace SqlStreamStore +{ + using System; + using System.Threading.Tasks; + using Npgsql; + using SqlStreamStore.Infrastructure; + using SqlStreamStore.TestUtils.Postgres; + + public class PostgresStreamStoreFixture : IStreamStoreFixture + { + private readonly Action _onDispose; + private bool _preparedPreviously; + private readonly PostgresStreamStoreV2Settings _settings; + + public PostgresStreamStoreFixture( + string schema, + PostgresContainer dockerInstance, + string databaseName, + Action onDispose) + { + _onDispose = onDispose; + + DatabaseName = databaseName; + var connectionString = dockerInstance.ConnectionString; + + _settings = new PostgresStreamStoreV2Settings(connectionString) + { + Schema = schema, + GetUtcNow = () => GetUtcNow(), + DisableDeletionTracking = false, + ScavengeAsynchronously = false, + }; + } + + public void Dispose() + { + Store.Dispose(); + PostgresStreamStore = null; + _onDispose(); + } + + public string DatabaseName { get; } + + public IStreamStore Store => PostgresStreamStore; + + public PostgresStreamStoreV2 PostgresStreamStore { get; private set; } + + public GetUtcNow GetUtcNow { get; set; } = SystemClock.GetUtcNow; + + public long MinPosition { get; set; } = 0; + + public int MaxSubscriptionCount { get; set; } = 100; + + public bool DisableDeletionTracking + { + get => _settings.DisableDeletionTracking; + set => _settings.DisableDeletionTracking = value; + } + + public async Task Prepare() + { + _settings.DisableDeletionTracking = false; + PostgresStreamStore = new PostgresStreamStoreV2(_settings); + + await PostgresStreamStore.CreateSchemaIfNotExists(); + if (_preparedPreviously) + { + using (var connection = new NpgsqlConnection(_settings.ConnectionString)) + { + connection.Open(); + + var schema = _settings.Schema; + + var commandText = $"DELETE FROM {schema}.messages"; + using (var command = new NpgsqlCommand(commandText, connection)) + { + command.ExecuteNonQuery(); + } + + commandText = $"DELETE FROM {schema}.streams"; + using (var command = new NpgsqlCommand(commandText, connection)) + { + command.ExecuteNonQuery(); + } + + commandText = $"ALTER SEQUENCE {schema}.streams_seq RESTART WITH 1;"; + using (var command = new NpgsqlCommand(commandText, connection)) + { + command.ExecuteNonQuery(); + } + + commandText = $"ALTER SEQUENCE {schema}.messages_seq RESTART WITH 0;"; + using (var command = new NpgsqlCommand(commandText, connection)) + { + command.ExecuteNonQuery(); + } + } + } + + _preparedPreviously = true; + } + } +} \ No newline at end of file diff --git a/tests/SqlStreamStore.Postgres.V2.Tests/PostgresStreamStoreFixturePool.cs b/tests/SqlStreamStore.Postgres.V2.Tests/PostgresStreamStoreFixturePool.cs new file mode 100644 index 000000000..ba5ed6f85 --- /dev/null +++ b/tests/SqlStreamStore.Postgres.V2.Tests/PostgresStreamStoreFixturePool.cs @@ -0,0 +1,58 @@ +namespace SqlStreamStore +{ + using System; + using System.Collections.Concurrent; + using System.Threading.Tasks; + using SqlStreamStore.TestUtils.Postgres; + using Xunit; + using Xunit.Abstractions; + + public class PostgresStreamStoreV3FixturePool : IAsyncLifetime + { + private readonly ConcurrentDictionary> _fixturePoolBySchema + = new ConcurrentDictionary>(); + + public async Task Get( + ITestOutputHelper outputHelper, + string schema = "dbo") + { + var fixturePool = _fixturePoolBySchema.GetOrAdd( + schema, + _ => new ConcurrentQueue()); + + if (!fixturePool.TryDequeue(out var fixture)) + { + var databaseName = $"test_{Guid.NewGuid():n}"; + var dockerInstance = new PostgresContainer(databaseName); + await dockerInstance.Start(); + await dockerInstance.CreateDatabase(); + + fixture = new PostgresStreamStoreFixture( + schema, + dockerInstance, + databaseName, + onDispose:() => fixturePool.Enqueue(fixture)); + + outputHelper.WriteLine($"Using new fixture with db {databaseName}"); + } + else + { + outputHelper.WriteLine($"Using pooled fixture with db {fixture.DatabaseName}"); + } + + await fixture.Prepare(); + + return fixture; + } + + public Task InitializeAsync() + { + return Task.CompletedTask; + } + + public Task DisposeAsync() + { + return Task.CompletedTask; + } + } +} \ No newline at end of file diff --git a/tests/SqlStreamStore.Postgres.V2.Tests/SqlStreamStore.Postgres.V2.Tests.csproj b/tests/SqlStreamStore.Postgres.V2.Tests/SqlStreamStore.Postgres.V2.Tests.csproj new file mode 100644 index 000000000..ab874c70c --- /dev/null +++ b/tests/SqlStreamStore.Postgres.V2.Tests/SqlStreamStore.Postgres.V2.Tests.csproj @@ -0,0 +1,16 @@ + + + + net6.0 + SqlStreamStore.Postgres.V2.Tests + SqlStreamStore + + + + + + + + + + \ No newline at end of file diff --git a/tests/SqlStreamStore.Sqlite.Tests/SqlStreamStore.Sqlite.Tests.csproj b/tests/SqlStreamStore.Sqlite.Tests/SqlStreamStore.Sqlite.Tests.csproj index 6c5ebb697..5cc1e46eb 100644 --- a/tests/SqlStreamStore.Sqlite.Tests/SqlStreamStore.Sqlite.Tests.csproj +++ b/tests/SqlStreamStore.Sqlite.Tests/SqlStreamStore.Sqlite.Tests.csproj @@ -1,7 +1,7 @@ - netcoreapp3.1 - SqlStreamStore.Sqlite.Tests + net6.0 + SqlStreamStore.Sqlite.Tests SqlStreamStore @@ -11,5 +11,4 @@ - diff --git a/tests/SqlStreamStore.Tests/SqlStreamStore.Tests.csproj b/tests/SqlStreamStore.Tests/SqlStreamStore.Tests.csproj index d49b84447..f472007dc 100644 --- a/tests/SqlStreamStore.Tests/SqlStreamStore.Tests.csproj +++ b/tests/SqlStreamStore.Tests/SqlStreamStore.Tests.csproj @@ -1,6 +1,6 @@  - netcoreapp3.1 + net6.0 SqlStreamStore.Tests SqlStreamStore diff --git a/tools/hal-docs/gen-hal-docs.ps1 b/tools/hal-docs/gen-hal-docs.ps1 index 2c9c3f455..eb36aae4d 100644 --- a/tools/hal-docs/gen-hal-docs.ps1 +++ b/tools/hal-docs/gen-hal-docs.ps1 @@ -3,5 +3,5 @@ pushd cd ../../ -dotnet run -p build/build.csproj -- build-hal-docs +dotnet run --project build/build.csproj -- build-hal-docs popd \ No newline at end of file From 31fbee319d6a5cc82aa0670a0ef5e080ef543904 Mon Sep 17 00:00:00 2001 From: Arne Schoonvliet Date: Tue, 15 Feb 2022 08:44:15 +0100 Subject: [PATCH 07/13] Push to correct new feedz --- build/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Program.cs b/build/Program.cs index b230dbea8..5edd76a46 100644 --- a/build/Program.cs +++ b/build/Program.cs @@ -154,7 +154,7 @@ void RunTest(string project) foreach (var packageToPush in packagesToPush) { - Run("dotnet", $"nuget push {packageToPush} -s https://f.feedz.io/logicality/streamstore-ci/nuget/index.json -k {apiKey} --skip-duplicate", noEcho: true); + Run("dotnet", $"nuget push {packageToPush} -s https://f.feedz.io/gritti/sqlstreamstore/nuget/index.json -k {apiKey} --skip-duplicate", noEcho: true); } }); From c380e16831aa25bfd199ba8d0f3d6249471c5fa1 Mon Sep 17 00:00:00 2001 From: Arne Schoonvliet Date: Wed, 2 Mar 2022 13:18:18 +0100 Subject: [PATCH 08/13] Upgrade nuget packages (#2) * Upgrade nuget packages * Use correct version tavis Co-authored-by: Arne Schoonvliet --- src/Directory.Build.props | 2 +- .../SqlStreamStore.HAL.DevServer.csproj | 2 +- src/SqlStreamStore.MySql/SqlStreamStore.MySql.csproj | 2 +- src/SqlStreamStore.TestUtils/SqlStreamStore.TestUtils.csproj | 4 ++-- tests/Directory.Build.props | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Directory.Build.props b/src/Directory.Build.props index 310b95e4e..6502e1d44 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -16,7 +16,7 @@ - + diff --git a/src/SqlStreamStore.HAL.DevServer/SqlStreamStore.HAL.DevServer.csproj b/src/SqlStreamStore.HAL.DevServer/SqlStreamStore.HAL.DevServer.csproj index d3925740a..548b38fa1 100644 --- a/src/SqlStreamStore.HAL.DevServer/SqlStreamStore.HAL.DevServer.csproj +++ b/src/SqlStreamStore.HAL.DevServer/SqlStreamStore.HAL.DevServer.csproj @@ -10,7 +10,7 @@ - + diff --git a/src/SqlStreamStore.MySql/SqlStreamStore.MySql.csproj b/src/SqlStreamStore.MySql/SqlStreamStore.MySql.csproj index 01afcb004..7a057be2b 100644 --- a/src/SqlStreamStore.MySql/SqlStreamStore.MySql.csproj +++ b/src/SqlStreamStore.MySql/SqlStreamStore.MySql.csproj @@ -12,7 +12,7 @@ - + diff --git a/src/SqlStreamStore.TestUtils/SqlStreamStore.TestUtils.csproj b/src/SqlStreamStore.TestUtils/SqlStreamStore.TestUtils.csproj index 76afc7b59..59e3a8f16 100644 --- a/src/SqlStreamStore.TestUtils/SqlStreamStore.TestUtils.csproj +++ b/src/SqlStreamStore.TestUtils/SqlStreamStore.TestUtils.csproj @@ -8,8 +8,8 @@ - - + + diff --git a/tests/Directory.Build.props b/tests/Directory.Build.props index 2029261a7..6536bb510 100644 --- a/tests/Directory.Build.props +++ b/tests/Directory.Build.props @@ -7,7 +7,7 @@ - + From a38b68362ff04de04955c3e37f8bb49d33c1215e Mon Sep 17 00:00:00 2001 From: Arne Schoonvliet Date: Thu, 3 Mar 2022 12:13:12 +0100 Subject: [PATCH 09/13] Replace functions in migration to have correct type (#3) Co-authored-by: Arne Schoonvliet --- .../PgSqlScriptsV2/Migration.sql | 390 +++++++++++++++++- 1 file changed, 389 insertions(+), 1 deletion(-) diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Migration.sql b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Migration.sql index 6adeb62ae..914dcbcad 100644 --- a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Migration.sql +++ b/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Migration.sql @@ -1,3 +1,391 @@ SET TimeZone='UTC'; ALTER TABLE __schema__.messages ALTER COLUMN "created_utc" TYPE TIMESTAMP WITH TIME ZONE; -COMMENT ON SCHEMA __schema__ IS '{ "version": 2 }'; \ No newline at end of file +COMMENT ON SCHEMA __schema__ IS '{ "version": 2 }'; + +CREATE OR REPLACE FUNCTION __schema__.append_to_stream( + _stream_id CHAR(42), + _stream_id_original VARCHAR(1000), + _metadata_stream_id CHAR(42), + _expected_version INT, + _created_utc TIMESTAMP WITH TIME ZONE, + _new_stream_messages __schema__.new_stream_message []) + RETURNS TABLE( + current_version INT, + current_position BIGINT + ) AS $F$ +DECLARE + _current_version INT; + _current_position BIGINT; + _stream_id_internal INT; + _success INT; + _max_age INT; + _max_count INT; +BEGIN + IF _created_utc IS NULL + THEN + _created_utc = now() at time zone 'utc'; + END IF; + + IF _expected_version < 0 + THEN + SELECT __schema__.messages.json_data :: JSON->>'MaxAge', __schema__.messages.json_data :: JSON->>'MaxCount' + INTO _max_age, _max_count + FROM __schema__.messages + JOIN __schema__.streams ON __schema__.messages.stream_id_internal = __schema__.streams.id_internal + WHERE __schema__.streams.id = _metadata_stream_id + ORDER BY __schema__.messages.stream_version DESC + LIMIT 1; + + INSERT INTO __schema__.streams (id, id_original, max_age, max_count) + SELECT _stream_id, _stream_id_original, _max_age, _max_count + ON CONFLICT DO NOTHING; + GET DIAGNOSTICS _success = ROW_COUNT; + + END IF; + + IF _expected_version = -1 /* ExpectedVersion.Empty */ + THEN + + IF _success = 0 AND + cardinality(_new_stream_messages) > 0 AND + (SELECT __schema__.streams.version FROM __schema__.streams WHERE __schema__.streams.id_internal = _stream_id_internal) > 0 + THEN + + RAISE EXCEPTION 'WrongExpectedVersion'; + END IF; + ELSIF _expected_version = -3 /* ExpectedVersion.NoStream */ + THEN + IF _success = 0 AND cardinality(_new_stream_messages) > 0 + THEN + PERFORM __schema__.enforce_idempotent_append( + _stream_id, + 0, + false, + _new_stream_messages); + SELECT version, position, id_internal + INTO _current_version, _current_position, _stream_id_internal + FROM __schema__.streams + WHERE __schema__.streams.id = _stream_id; + + RETURN QUERY + SELECT _current_version, _current_position; + RETURN; + END IF; + END IF; + + SELECT (CASE _expected_version + WHEN -2 /* ExpectedVersion.Any */ + THEN coalesce( + __schema__.read_stream_version_of_message_id( + __schema__.streams.id_internal, + _new_stream_messages [ 1 ].message_id) - 1, + __schema__.streams.version) + WHEN -3 THEN -1 /* ExpectedVersion.NoStream */ + WHEN -1 THEN -1 /* ExpectedVersion.Empty */ + ELSE _expected_version END), __schema__.streams.position, __schema__.streams.id_internal + INTO _current_version, _current_position, _stream_id_internal + FROM __schema__.streams + WHERE __schema__.streams.id = _stream_id; + + IF (_expected_version >= 0 AND ( + SELECT __schema__.streams.version + FROM __schema__.streams + WHERE __schema__.streams.id_internal = _stream_id_internal + ) < _expected_version) + THEN + RAISE EXCEPTION 'WrongExpectedVersion'; + END IF; + + IF (_expected_version >= 0 AND _stream_id_internal IS NULL) + THEN + RAISE EXCEPTION 'WrongExpectedVersion'; + END IF; + + IF cardinality(_new_stream_messages) > 0 + THEN + INSERT INTO __schema__.messages (message_id, + stream_id_internal, + stream_version, + created_utc, + type, + json_data, + json_metadata) + SELECT m.message_id, _stream_id_internal, _current_version + (row_number() + over ()) :: int, _created_utc, m.type, m.json_data, m.json_metadata + FROM unnest(_new_stream_messages) m + ON CONFLICT DO NOTHING; + GET DIAGNOSTICS _success = ROW_COUNT; + + IF (_success <> cardinality(_new_stream_messages)) + THEN + IF (_expected_version = -2) /* ExpectedVersion.Any */ + THEN + + PERFORM __schema__.enforce_idempotent_append( + _stream_id, + _current_version + 1 - _success, + false, + _new_stream_messages); + + ELSEIF _expected_version = -3 /* ExpectedVersion.NoStream */ + THEN + RAISE EXCEPTION 'WhyAreYouHere'; /* there is no way to get here? */ + ELSEIF _expected_version = -1 /* ExpectedVersion.Empty */ + THEN + PERFORM __schema__.enforce_idempotent_append( + _stream_id, + 0, + false, + _new_stream_messages); + ELSE + PERFORM __schema__.enforce_idempotent_append( + _stream_id, + _expected_version + 1 - _success, + true, + _new_stream_messages); + SELECT version, position, id_internal + INTO _current_version, _current_position, _stream_id_internal + FROM __schema__.streams + WHERE __schema__.streams.id = _stream_id; + + RETURN QUERY + SELECT _current_version, _current_position; + RETURN; + END IF; + END IF; + + SELECT COALESCE(__schema__.messages.position, -1), COALESCE(__schema__.messages.stream_version, -1) + INTO _current_position, _current_version + FROM __schema__.messages + WHERE __schema__.messages.stream_id_internal = _stream_id_internal + ORDER BY __schema__.messages.position DESC + LIMIT 1; + + UPDATE __schema__.streams + SET "version" = _current_version, + "position" = _current_position + WHERE id_internal = _stream_id_internal; + + RETURN QUERY + SELECT _current_version, _current_position; + + ELSE + RETURN QUERY + SELECT -1, -1 :: BIGINT; + + END IF; + +END; + +$F$ +LANGUAGE 'plpgsql'; + + +CREATE OR REPLACE FUNCTION __schema__.delete_stream( + _stream_id CHAR(42), + _expected_version INT, + _created_utc TIMESTAMP WITH TIME ZONE, + _deletion_tracking_disabled BOOLEAN, + _deleted_stream_id CHAR(42), + _deleted_stream_id_original VARCHAR(1000), + _deleted_stream_message __schema__.new_stream_message) + RETURNS VOID +AS $F$ +DECLARE + _stream_id_internal INT; + _latest_stream_version INT; + _affected INT; +BEGIN + IF _created_utc IS NULL + THEN + _created_utc = now() at time zone 'utc'; + END IF; + SELECT __schema__.streams.id_internal + INTO _stream_id_internal + FROM __schema__.streams + WHERE __schema__.streams.id = _stream_id; + + IF _expected_version = -1 /* ExpectedVersion.NoStream */ + THEN + RAISE EXCEPTION 'WrongExpectedVersion'; + ELSIF _expected_version >= 0 /* ExpectedVersion */ + THEN + IF _stream_id_internal IS NULL + THEN + RAISE EXCEPTION 'WrongExpectedVersion'; + END IF; + + SELECT __schema__.messages.stream_version + INTO _latest_stream_version + FROM __schema__.messages + WHERE __schema__.messages.stream_id_internal = _stream_id_internal + ORDER BY __schema__.messages.position DESC + LIMIT 1; + + IF _latest_stream_version != _expected_version + THEN + RAISE EXCEPTION 'WrongExpectedVersion'; + END IF; + END IF; + + DELETE FROM __schema__.messages WHERE __schema__.messages.stream_id_internal = _stream_id_internal; + + DELETE FROM __schema__.streams WHERE __schema__.streams.id = _stream_id; + + IF (_deletion_tracking_disabled = TRUE) + THEN + RETURN; + END IF; + + GET DIAGNOSTICS _affected = ROW_COUNT; + + IF _affected > 0 + THEN + PERFORM __schema__.append_to_stream( + _deleted_stream_id, + _deleted_stream_id_original, + NULL, + -2, + _created_utc, + ARRAY [_deleted_stream_message]); + + END IF; +END; + +$F$ +LANGUAGE 'plpgsql'; + +CREATE OR REPLACE FUNCTION __schema__.delete_stream_messages( + _stream_id CHAR(42), + _message_ids UUID [], + _deletion_tracking_disabled BOOLEAN, + _deleted_stream_id CHAR(42), + _deleted_stream_id_original VARCHAR(1000), + _created_utc TIMESTAMP WITH TIME ZONE, + _deleted_messages __schema__.new_stream_message [] +) + RETURNS VOID +AS $F$ +DECLARE + _stream_id_internal INT; + _deleted_count NUMERIC; +BEGIN + IF _created_utc IS NULL THEN + _created_utc = now() at time zone 'utc'; + END IF; + + SELECT __schema__.streams.id_internal + INTO _stream_id_internal + FROM __schema__.streams + WHERE __schema__.streams.id = _stream_id; + + WITH deleted AS (DELETE FROM __schema__.messages + WHERE __schema__.messages.stream_id_internal = _stream_id_internal + AND __schema__.messages.message_id = ANY (_message_ids) + RETURNING *) + SELECT count(*) + FROM deleted + INTO _deleted_count; + + IF (_deletion_tracking_disabled = FALSE AND _deleted_count > 0) + THEN + PERFORM __schema__.append_to_stream( + _deleted_stream_id, + _deleted_stream_id_original, + NULL, + -2, + _created_utc, + _deleted_messages); + END IF; +END; + +$F$ +LANGUAGE 'plpgsql'; + +DROP FUNCTION public.read_all(int4, int8, bool, bool); +CREATE OR REPLACE FUNCTION __schema__.read_all( + _count INT, + _position BIGINT, + _forwards BOOLEAN, + _prefetch BOOLEAN +) + RETURNS TABLE( + stream_id VARCHAR(1000), + message_id UUID, + stream_version INT, + "position" BIGINT, + create_utc TIMESTAMP WITH TIME ZONE, + "type" VARCHAR(128), + json_metadata JSONB, + json_data JSONB, + max_age INT + ) +AS $F$ +BEGIN + + RETURN QUERY + WITH messages AS ( + SELECT __schema__.streams.id_original, + __schema__.messages.message_id, + __schema__.messages.stream_version, + __schema__.messages.position, + __schema__.messages.created_utc, + __schema__.messages.type, + __schema__.messages.json_metadata, + (CASE _prefetch + WHEN TRUE THEN __schema__.messages.json_data + ELSE NULL END), + __schema__.streams.max_age + FROM __schema__.messages + INNER JOIN __schema__.streams ON __schema__.messages.stream_id_internal = __schema__.streams.id_internal + WHERE (CASE + WHEN _forwards THEN __schema__.messages.position >= _position + ELSE __schema__.messages.position <= _position END) + ORDER BY + (CASE WHEN _forwards THEN __schema__.messages.position END), + (CASE WHEN not _forwards THEN __schema__.messages.position END) DESC + LIMIT _count + ) + SELECT * FROM messages LIMIT _count; +END; +$F$ +LANGUAGE 'plpgsql'; + +CREATE OR REPLACE FUNCTION __schema__.set_stream_metadata( + _stream_id CHAR(42), + _metadata_stream_id CHAR(42), + _metadata_stream_id_original CHAR(42), + _max_age INT, + _max_count INT, + _expected_version INT, + _created_utc TIMESTAMP WITH TIME ZONE, + _metadata_message __schema__.new_stream_message) + RETURNS INT AS $F$ +DECLARE + _current_version INT; +BEGIN + IF _created_utc IS NULL + THEN + _created_utc = now() at time zone 'utc'; + END IF; + + SELECT current_version + FROM __schema__.append_to_stream( + _metadata_stream_id, + _metadata_stream_id_original, + NULL, + _expected_version, + _created_utc, + ARRAY [_metadata_message] + ) + INTO _current_version; + + UPDATE __schema__.streams + SET max_age = _max_age, + max_count = _max_count + WHERE __schema__.streams.id = _stream_id; + + RETURN _current_version; +END; +$F$ +LANGUAGE 'plpgsql'; \ No newline at end of file From f5376cf061e793bdf866a4eda8db1e919fc4a6cc Mon Sep 17 00:00:00 2001 From: Arne Schoonvliet Date: Wed, 20 Apr 2022 14:10:56 +0200 Subject: [PATCH 10/13] WIP --- .../Postgres/PostgresContainer.cs | 73 ++++++++++---- .../Postgres/PostgresDatabaseManager.cs | 4 +- .../AcceptanceTests.DeletionTracking.cs | 2 +- .../AcceptanceTests.cs | 3 +- .../PostgresStreamStoreFixture.cs | 2 +- .../PostgresStreamStoreFixturePool.cs | 2 +- .../PostgresStreamStoreTests.cs | 96 +++++++++++++++++++ 7 files changed, 160 insertions(+), 22 deletions(-) create mode 100644 tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreTests.cs diff --git a/src/SqlStreamStore.TestUtils/Postgres/PostgresContainer.cs b/src/SqlStreamStore.TestUtils/Postgres/PostgresContainer.cs index 02554f23c..61cb74240 100644 --- a/src/SqlStreamStore.TestUtils/Postgres/PostgresContainer.cs +++ b/src/SqlStreamStore.TestUtils/Postgres/PostgresContainer.cs @@ -1,9 +1,12 @@ namespace SqlStreamStore.TestUtils.Postgres { using System; + using System.Globalization; + using System.Linq; using System.Threading; using System.Threading.Tasks; - using Ductus.FluentDocker.Builders; + //using Ductus.FluentDocker.Builders; + using Ductus.FluentDocker.Model.Containers; using Ductus.FluentDocker.Services; using Npgsql; using Polly; @@ -13,23 +16,52 @@ public class PostgresContainer : PostgresDatabaseManager private readonly IContainerService _containerService; private const string Image = "postgres:10.4-alpine"; private const string ContainerName = "sql-stream-store-tests-postgres"; - private const int Port = 5432; - public override string ConnectionString => ConnectionStringBuilder.ConnectionString; + private readonly int _port; + //public override string ConnectionString => ConnectionStringBuilder.ConnectionString; - public PostgresContainer(string databaseName) + + public PostgresContainer(string schema, string databaseName, float cpu = float.MinValue) : base(databaseName) { - - _containerService = new Builder() - .UseContainer() - .WithName(ContainerName) - .UseImage(Image) - .KeepRunning() - .ReuseIfExists() - .ExposePort(Port, Port) - .Command("-N", "500") - .Build(); + CultureInfo.DefaultThreadCurrentCulture = new CultureInfo("en-US"); + + var hosts = new Hosts().Discover(); + var host = hosts.FirstOrDefault(x => x.IsNative) ?? hosts.FirstOrDefault(x => x.Name == "default"); + + + var b = host.GetRunningContainers().Select(x => x.GetConfiguration(true)) + .Select(x => x.NetworkSettings.Ports).SelectMany(x => x.Values).Where(x => x is not null && x.Any()).SelectMany(x => x).Select(x => x.Port).ToList(); + + var possiblePort = 5432; + while (b.Contains(possiblePort)) + { + possiblePort++; + } + _port = possiblePort; + + _containerService = host.Create(Image, false, new ContainerCreateParams + { + Cpus = cpu, + Name = $"{schema}-{ContainerName}", + PortMappings = new[] { $"{_port}:5432" } + + }, false, true, command: "-N 500"); + + + //_containerService = new Builder() + // .UseContainer() + // .WithName(ContainerName) + // .UseImage(Image) + // .KeepRunning() + // .ReuseIfExists() + // //.ExposePort(Port, Port) + // .Command("-N", "500") + // .WithParentCGroup() + // .Build(); + + //var c = _containerService.GetConfiguration(true); + //var d = c1.GetConfiguration(true); } public async Task Start(CancellationToken cancellationToken = default) @@ -38,7 +70,7 @@ public async Task Start(CancellationToken cancellationToken = default) await Policy .Handle() - .WaitAndRetryAsync(30, _ => TimeSpan.FromMilliseconds(500)) + .WaitAndRetryAsync(100, _ => TimeSpan.FromMilliseconds(500)) .ExecuteAsync(async () => { using(var connection = new NpgsqlConnection(DefaultConnectionString)) @@ -48,17 +80,26 @@ await Policy }); } + public override string GenerateConnectionString(string applicationName = "default") + { + return new NpgsqlConnectionStringBuilder(ConnectionStringBuilder.ConnectionString) + { + ApplicationName = applicationName + }.ConnectionString; + } + private NpgsqlConnectionStringBuilder ConnectionStringBuilder => new NpgsqlConnectionStringBuilder { Database = DatabaseName, Password = Environment.OSVersion.IsWindows() ? "password" : null, - Port = Port, + Port = _port, Username = "postgres", Host = "localhost", Pooling = true, MaxPoolSize = 1024, + Timeout = 300, IncludeErrorDetails = false }; } diff --git a/src/SqlStreamStore.TestUtils/Postgres/PostgresDatabaseManager.cs b/src/SqlStreamStore.TestUtils/Postgres/PostgresDatabaseManager.cs index d2f739575..2b3db4ca2 100644 --- a/src/SqlStreamStore.TestUtils/Postgres/PostgresDatabaseManager.cs +++ b/src/SqlStreamStore.TestUtils/Postgres/PostgresDatabaseManager.cs @@ -11,12 +11,12 @@ public abstract class PostgresDatabaseManager private bool _started; - protected string DefaultConnectionString => new NpgsqlConnectionStringBuilder(ConnectionString) + protected string DefaultConnectionString => new NpgsqlConnectionStringBuilder(GenerateConnectionString()) { Database = null }.ConnectionString; - public abstract string ConnectionString { get; } + public abstract string GenerateConnectionString(string applicationName = "default"); static PostgresDatabaseManager() { diff --git a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.DeletionTracking.cs b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.DeletionTracking.cs index 4e23ece16..852ecafcd 100644 --- a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.DeletionTracking.cs +++ b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.DeletionTracking.cs @@ -25,7 +25,7 @@ public async Task When_deletion_tracking_is_disabled_deleted_message_should_not_ public async Task When_deletion_tracking_is_disabled_deleted_stream_should_not_be_tracked() { Fixture.DisableDeletionTracking = true; - + var messages = CreateNewStreamMessages(1); await Store.AppendToStream("stream", ExpectedVersion.NoStream, messages); await Store.DeleteStream("stream"); diff --git a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.cs b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.cs index ee64f4ad7..7978f068c 100644 --- a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.cs +++ b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; + using MorseCode.ITask; using Shouldly; using SqlStreamStore.Streams; using SqlStreamStore.TestUtils; @@ -50,7 +51,7 @@ public async Task When_dispose_and_read_then_should_throw() { Store.Dispose(); - Func act = () => Store.ReadAllForwards(Position.Start, 10); + Func act = () => Store.ReadAllForwards(Position.Start, 10).AsTask(); await act.ShouldThrowAsync(); } diff --git a/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreFixture.cs b/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreFixture.cs index 3b7027c4c..16ad7f62a 100644 --- a/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreFixture.cs +++ b/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreFixture.cs @@ -21,7 +21,7 @@ public PostgresStreamStoreFixture( _onDispose = onDispose; DatabaseName = databaseName; - var connectionString = dockerInstance.ConnectionString; + var connectionString = dockerInstance.GenerateConnectionString(); _settings = new PostgresStreamStoreSettings(connectionString) { diff --git a/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreFixturePool.cs b/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreFixturePool.cs index ba5ed6f85..bd5ede374 100644 --- a/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreFixturePool.cs +++ b/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreFixturePool.cs @@ -23,7 +23,7 @@ public async Task Get( if (!fixturePool.TryDequeue(out var fixture)) { var databaseName = $"test_{Guid.NewGuid():n}"; - var dockerInstance = new PostgresContainer(databaseName); + var dockerInstance = new PostgresContainer(schema, databaseName); await dockerInstance.Start(); await dockerInstance.CreateDatabase(); diff --git a/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreTests.cs b/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreTests.cs new file mode 100644 index 000000000..a048e7137 --- /dev/null +++ b/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreTests.cs @@ -0,0 +1,96 @@ +namespace SqlStreamStore +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using Shouldly; + using SqlStreamStore.Infrastructure; + using SqlStreamStore.Streams; + using SqlStreamStore.TestUtils; + using SqlStreamStore.TestUtils.Postgres; + using Xunit; + using Xunit.Abstractions; + + public class PostgresStreamStoreTests : IDisposable + { + private readonly PostgresContainer _container; + private const string Schema = "stresstest"; + private readonly ITestOutputHelper _testOutputHelper; + private readonly IDisposable _logCapture; + + public PostgresStreamStoreTests(ITestOutputHelper testOutputHelper) + { + _container = new PostgresContainer(Schema, $"test_{Guid.NewGuid():n}", 0.02f); + _testOutputHelper = testOutputHelper; + _logCapture = LoggingHelper.Capture(testOutputHelper); + } + + [Fact] + public async Task Can_Handle_High_Load_Gaps() + { + await _container.Start(); + await _container.CreateDatabase(); + + var readerSettings = GetSettings(_container.GenerateConnectionString("reader")); + var writerSettings = GetSettings(_container.GenerateConnectionString("writer")); + + using var readerStore = new PostgresStreamStore(readerSettings); + using var writerStore = new PostgresStreamStore(writerSettings); + + await writerStore.CreateSchemaIfNotExists(); + + var cancellationTokenSource = new CancellationTokenSource(); + + var receiveMessages = new TaskCompletionSource(); + List receivedMessages = new List(); + + using var all = readerStore.SubscribeToAll( + Position.None, + (_, message, __) => + { + _testOutputHelper.WriteLine($"Received message {message.StreamId} " + + $"{message.StreamVersion} {message.Position}"); + if (message.Position >= 100) + receiveMessages.SetResult(message); + else + receivedMessages.Add(message); + + return Task.CompletedTask; + }); + + var appendTask = AppendMessages(writerStore, 200, cancellationTokenSource.Token); + await Task.WhenAny(receiveMessages.Task.WithTimeout(100000), appendTask); + cancellationTokenSource.Cancel(); + + receivedMessages.Count.ShouldBe(100); + } + + private static PostgresStreamStoreSettings GetSettings(string connectionString) + { + return new PostgresStreamStoreSettings(connectionString) + { + Schema = Schema, + GetUtcNow = SystemClock.GetUtcNow, + DisableDeletionTracking = false, + ScavengeAsynchronously = false + }; + } + + private static async Task AppendMessages(IStreamStore streamStore, int numberOfEvents, CancellationToken ct) + { + await Task.WhenAll(Enumerable.Range(0, numberOfEvents).Select(_ => + { + var newMessage = new NewStreamMessage(Guid.NewGuid(), "MyEvent", "{}"); + return streamStore.AppendToStream(Guid.NewGuid().ToString(), ExpectedVersion.Any, newMessage, ct); + })); + } + + public void Dispose() + { + _container.Dispose(); + _logCapture.Dispose(); + } + } +} \ No newline at end of file From 2bf093115c3a459bdf16feb715bded40d8d2b0ed Mon Sep 17 00:00:00 2001 From: Arne Schoonvliet Date: Wed, 20 Apr 2022 15:22:12 +0200 Subject: [PATCH 11/13] WIP --- SqlStreamStore.sln | 4 - build/build.csproj | 1 - src/LoadTests/LoadTests.csproj | 1 - .../SqlStreamStore.HAL.DevServer.csproj | 1 - .../SqlStreamStore.HAL.csproj | 1 - .../SqlStreamStore.Http.csproj | 2 +- .../MsSqlStreamStore.ReadAll.cs | 6 - .../MsSqlStreamStoreV3.ReadAll.cs | 6 - .../SqlStreamStore.MySql.csproj | 2 +- .../PgSqlScriptsV2/AppendToStream.sql | 0 .../PgSqlScriptsV2/DeleteStream.sql | 0 .../PgSqlScriptsV2/DeleteStreamMessages.sql | 0 .../PgSqlScriptsV2/DropAll.sql | 0 .../PgSqlScriptsV2/EnableExplainAnalyze.sql | 0 .../EnforceIdempotentAppend.sql | 0 .../PgSqlScriptsV2/ListStreams.sql | 0 .../PgSqlScriptsV2/ListStreamsEndingWith.sql | 0 .../ListStreamsStartingWith.sql | 0 .../PgSqlScriptsV2/Migration.sql | 0 .../PgSqlScriptsV2/Parameters.cs | 0 .../PgSqlScriptsV2/Read.sql | 0 .../PgSqlScriptsV2/ReadAll.sql | 0 .../PgSqlScriptsV2/ReadHeadPosition.sql | 0 .../PgSqlScriptsV2/ReadJsonData.sql | 0 .../PgSqlScriptsV2/ReadSchemaVersion.sql | 0 .../PgSqlScriptsV2/ReadStreamHeadPosition.sql | 0 .../PgSqlScriptsV2/ReadStreamHeadVersion.sql | 0 .../ReadStreamVersionOfMessageId.sql | 0 .../PgSqlScriptsV2/Scavenge.sql | 0 .../PgSqlScriptsV2/Schema.cs | 0 .../PgSqlScriptsV2/Scripts.cs | 0 .../PgSqlScriptsV2/SetStreamMetadata.sql | 0 .../PgSqlScriptsV2/Tables.sql | 0 .../PostgresStreamStore.Subscriptions.cs | 1 - .../PostgresStreamStore.cs | 175 +++++------ .../PostgresStreamStoreSettings.cs | 1 - .../SqlStreamStore.Postgres.csproj | 6 +- .../PostgresStreamStoreV2.Append.cs | 91 ------ .../PostgresStreamStoreV2.Delete.cs | 111 ------- .../PostgresStreamStoreV2.ListStreams.cs | 77 ----- .../PostgresStreamStoreV2.Metadata.cs | 101 ------- .../PostgresStreamStoreV2.Read.cs | 278 ------------------ .../PostgresStreamStoreV2.ReadAll.cs | 212 ------------- .../PostgresStreamStoreV2.Subscriptions.cs | 47 --- .../PostgresStreamStoreV2.cs | 272 ----------------- .../PostgresStreamStoreV2Settings.cs | 100 ------- .../Properties/AssemblyInfo.cs | 3 - .../SqlStreamStore.PostgresV2.csproj | 20 -- ...treamStore.SchemaCreationScriptTool.csproj | 1 - .../SqlStreamStore.Sqlite.csproj | 2 +- .../SqlStreamStore.TestUtils.csproj | 1 - src/SqlStreamStore/SqlStreamStore.csproj | 2 +- .../AcceptanceTests.cs | 8 +- .../IStreamStoreFixture.cs | 5 +- .../SqlStreamStore.HAL.Tests.csproj | 1 - .../SqlStreamStore.Http.Tests.csproj | 1 - .../MsSqlStreamStoreAcceptanceTests.cs | 5 +- .../MsSqlStreamStoreFixture.cs | 5 +- .../SqlStreamStore.MsSql.Tests.csproj | 1 - .../MsSqlStreamStoreV3AcceptanceTests.cs | 4 +- .../MsSqlStreamStoreV3Fixture.cs | 5 +- .../SqlStreamStore.MsSql.V3.Tests.csproj | 1 - .../SqlStreamStore.MySql.Tests.csproj | 1 - .../PostgresStreamStoreAcceptanceTests.cs | 5 +- .../PostgresStreamStoreFixture.cs | 2 +- .../PostgresStreamStoreFixturePool.cs | 2 +- .../SqlStreamStore.Postgres.Tests.csproj | 1 - .../PostgresStreamStoreAcceptanceTests.cs | 157 ---------- .../PostgresStreamStoreFixture.cs | 103 ------- .../PostgresStreamStoreFixturePool.cs | 58 ---- .../SqlStreamStore.Postgres.V2.Tests.csproj | 16 - .../SqlStreamStore.Sqlite.Tests.csproj | 3 +- 72 files changed, 97 insertions(+), 1812 deletions(-) rename src/{SqlStreamStore.PostgresV2 => SqlStreamStore.Postgres}/PgSqlScriptsV2/AppendToStream.sql (100%) rename src/{SqlStreamStore.PostgresV2 => SqlStreamStore.Postgres}/PgSqlScriptsV2/DeleteStream.sql (100%) rename src/{SqlStreamStore.PostgresV2 => SqlStreamStore.Postgres}/PgSqlScriptsV2/DeleteStreamMessages.sql (100%) rename src/{SqlStreamStore.PostgresV2 => SqlStreamStore.Postgres}/PgSqlScriptsV2/DropAll.sql (100%) rename src/{SqlStreamStore.PostgresV2 => SqlStreamStore.Postgres}/PgSqlScriptsV2/EnableExplainAnalyze.sql (100%) rename src/{SqlStreamStore.PostgresV2 => SqlStreamStore.Postgres}/PgSqlScriptsV2/EnforceIdempotentAppend.sql (100%) rename src/{SqlStreamStore.PostgresV2 => SqlStreamStore.Postgres}/PgSqlScriptsV2/ListStreams.sql (100%) rename src/{SqlStreamStore.PostgresV2 => SqlStreamStore.Postgres}/PgSqlScriptsV2/ListStreamsEndingWith.sql (100%) rename src/{SqlStreamStore.PostgresV2 => SqlStreamStore.Postgres}/PgSqlScriptsV2/ListStreamsStartingWith.sql (100%) rename src/{SqlStreamStore.PostgresV2 => SqlStreamStore.Postgres}/PgSqlScriptsV2/Migration.sql (100%) rename src/{SqlStreamStore.PostgresV2 => SqlStreamStore.Postgres}/PgSqlScriptsV2/Parameters.cs (100%) rename src/{SqlStreamStore.PostgresV2 => SqlStreamStore.Postgres}/PgSqlScriptsV2/Read.sql (100%) rename src/{SqlStreamStore.PostgresV2 => SqlStreamStore.Postgres}/PgSqlScriptsV2/ReadAll.sql (100%) rename src/{SqlStreamStore.PostgresV2 => SqlStreamStore.Postgres}/PgSqlScriptsV2/ReadHeadPosition.sql (100%) rename src/{SqlStreamStore.PostgresV2 => SqlStreamStore.Postgres}/PgSqlScriptsV2/ReadJsonData.sql (100%) rename src/{SqlStreamStore.PostgresV2 => SqlStreamStore.Postgres}/PgSqlScriptsV2/ReadSchemaVersion.sql (100%) rename src/{SqlStreamStore.PostgresV2 => SqlStreamStore.Postgres}/PgSqlScriptsV2/ReadStreamHeadPosition.sql (100%) rename src/{SqlStreamStore.PostgresV2 => SqlStreamStore.Postgres}/PgSqlScriptsV2/ReadStreamHeadVersion.sql (100%) rename src/{SqlStreamStore.PostgresV2 => SqlStreamStore.Postgres}/PgSqlScriptsV2/ReadStreamVersionOfMessageId.sql (100%) rename src/{SqlStreamStore.PostgresV2 => SqlStreamStore.Postgres}/PgSqlScriptsV2/Scavenge.sql (100%) rename src/{SqlStreamStore.PostgresV2 => SqlStreamStore.Postgres}/PgSqlScriptsV2/Schema.cs (100%) rename src/{SqlStreamStore.PostgresV2 => SqlStreamStore.Postgres}/PgSqlScriptsV2/Scripts.cs (100%) rename src/{SqlStreamStore.PostgresV2 => SqlStreamStore.Postgres}/PgSqlScriptsV2/SetStreamMetadata.sql (100%) rename src/{SqlStreamStore.PostgresV2 => SqlStreamStore.Postgres}/PgSqlScriptsV2/Tables.sql (100%) delete mode 100644 src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Append.cs delete mode 100644 src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Delete.cs delete mode 100644 src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.ListStreams.cs delete mode 100644 src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Metadata.cs delete mode 100644 src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Read.cs delete mode 100644 src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.ReadAll.cs delete mode 100644 src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Subscriptions.cs delete mode 100644 src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.cs delete mode 100644 src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2Settings.cs delete mode 100644 src/SqlStreamStore.PostgresV2/Properties/AssemblyInfo.cs delete mode 100644 src/SqlStreamStore.PostgresV2/SqlStreamStore.PostgresV2.csproj delete mode 100644 tests/SqlStreamStore.Postgres.V2.Tests/PostgresStreamStoreAcceptanceTests.cs delete mode 100644 tests/SqlStreamStore.Postgres.V2.Tests/PostgresStreamStoreFixture.cs delete mode 100644 tests/SqlStreamStore.Postgres.V2.Tests/PostgresStreamStoreFixturePool.cs delete mode 100644 tests/SqlStreamStore.Postgres.V2.Tests/SqlStreamStore.Postgres.V2.Tests.csproj diff --git a/SqlStreamStore.sln b/SqlStreamStore.sln index 94cef37f9..05d74e3a9 100644 --- a/SqlStreamStore.sln +++ b/SqlStreamStore.sln @@ -62,10 +62,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{845ED623-9F0 EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "build", "build\build.csproj", "{5273D70A-96D7-4775-B67C-BDE48D20CA36}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SqlStreamStore.Postgres.V2.Tests", "tests\SqlStreamStore.Postgres.V2.Tests\SqlStreamStore.Postgres.V2.Tests.csproj", "{A8D87EDC-E911-42F3-891F-82DEBBD4E0A3}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SqlStreamStore.PostgresV2", "src\SqlStreamStore.PostgresV2\SqlStreamStore.PostgresV2.csproj", "{D1FA4E2A-9113-4F8E-987D-DDC1C1C583E2}" -EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU diff --git a/build/build.csproj b/build/build.csproj index 85defe07c..b6b91514b 100644 --- a/build/build.csproj +++ b/build/build.csproj @@ -2,7 +2,6 @@ Exe - net5.0 net6.0 diff --git a/src/LoadTests/LoadTests.csproj b/src/LoadTests/LoadTests.csproj index 3de799420..b7ca31a30 100644 --- a/src/LoadTests/LoadTests.csproj +++ b/src/LoadTests/LoadTests.csproj @@ -1,6 +1,5 @@  - net5.0 net6.0 win10-x64;osx.10.11-x64;ubuntu.16.10-x64;ubuntu.14.04-x64 exe diff --git a/src/SqlStreamStore.HAL.DevServer/SqlStreamStore.HAL.DevServer.csproj b/src/SqlStreamStore.HAL.DevServer/SqlStreamStore.HAL.DevServer.csproj index 61b303105..6c25c05b8 100644 --- a/src/SqlStreamStore.HAL.DevServer/SqlStreamStore.HAL.DevServer.csproj +++ b/src/SqlStreamStore.HAL.DevServer/SqlStreamStore.HAL.DevServer.csproj @@ -1,7 +1,6 @@  Exe - net5.0 true false net6.0 diff --git a/src/SqlStreamStore.HAL/SqlStreamStore.HAL.csproj b/src/SqlStreamStore.HAL/SqlStreamStore.HAL.csproj index 2af084ece..538ad4b33 100644 --- a/src/SqlStreamStore.HAL/SqlStreamStore.HAL.csproj +++ b/src/SqlStreamStore.HAL/SqlStreamStore.HAL.csproj @@ -1,6 +1,5 @@  - net5.0 $(DefineConstants);LIBLOG_PORTABLE net5.0;net6.0 HTTP Server for SQL Stream Store diff --git a/src/SqlStreamStore.Http/SqlStreamStore.Http.csproj b/src/SqlStreamStore.Http/SqlStreamStore.Http.csproj index 3681269c7..ab448f83d 100644 --- a/src/SqlStreamStore.Http/SqlStreamStore.Http.csproj +++ b/src/SqlStreamStore.Http/SqlStreamStore.Http.csproj @@ -2,7 +2,7 @@ Http (HAL) provider for SQL Stream Store SQL Stream Store - Http (HAL) Provider - net5.0 + netstandard2.0 true SqlStreamStore.Http SqlStreamStore.Http diff --git a/src/SqlStreamStore.MsSql/MsSqlStreamStore.ReadAll.cs b/src/SqlStreamStore.MsSql/MsSqlStreamStore.ReadAll.cs index 3ec7c478c..c176f0cbd 100644 --- a/src/SqlStreamStore.MsSql/MsSqlStreamStore.ReadAll.cs +++ b/src/SqlStreamStore.MsSql/MsSqlStreamStore.ReadAll.cs @@ -15,7 +15,6 @@ protected override async Task ReadAllForwardsInternal( long fromPosition, int maxCount, bool prefetch, - ReadNextAllPage readNext, CancellationToken cancellationToken, long fromMaxPositionInclusive = -1) { @@ -44,7 +43,6 @@ protected override async Task ReadAllForwardsInternal( fromPosition, true, ReadDirection.Forward, - readNext, messages.ToArray()); } @@ -104,7 +102,6 @@ protected override async Task ReadAllForwardsInternal( nextPosition, isEnd, ReadDirection.Forward, - readNext, messages.ToArray()); } } @@ -114,7 +111,6 @@ protected override async Task ReadAllBackwardsInternal( long fromPositionExclusive, int maxCount, bool prefetch, - ReadNextAllPage readNext, CancellationToken cancellationToken) { maxCount = maxCount == int.MaxValue ? maxCount - 1 : maxCount; @@ -144,7 +140,6 @@ protected override async Task ReadAllBackwardsInternal( Position.Start, true, ReadDirection.Backward, - readNext, messages.ToArray()); } @@ -200,7 +195,6 @@ protected override async Task ReadAllBackwardsInternal( nextPosition, isEnd, ReadDirection.Backward, - readNext, messages.ToArray()); } } diff --git a/src/SqlStreamStore.MsSql/MsSqlStreamStoreV3.ReadAll.cs b/src/SqlStreamStore.MsSql/MsSqlStreamStoreV3.ReadAll.cs index 0ac0b081c..926195464 100644 --- a/src/SqlStreamStore.MsSql/MsSqlStreamStoreV3.ReadAll.cs +++ b/src/SqlStreamStore.MsSql/MsSqlStreamStoreV3.ReadAll.cs @@ -15,7 +15,6 @@ protected override async Task ReadAllForwardsInternal( long fromPosition, int maxCount, bool prefetch, - ReadNextAllPage readNext, CancellationToken cancellationToken, long fromMaxPositionInclusive = -1) { @@ -43,7 +42,6 @@ protected override async Task ReadAllForwardsInternal( fromPosition, true, ReadDirection.Forward, - readNext, Array.Empty()); } @@ -102,7 +100,6 @@ protected override async Task ReadAllForwardsInternal( nextPosition, isEnd, ReadDirection.Forward, - readNext, filteredMessages.ToArray()); } } @@ -112,7 +109,6 @@ protected override async Task ReadAllBackwardsInternal( long fromPositionExclusive, int maxCount, bool prefetch, - ReadNextAllPage readNext, CancellationToken cancellationToken) { maxCount = maxCount == int.MaxValue ? maxCount - 1 : maxCount; @@ -142,7 +138,6 @@ protected override async Task ReadAllBackwardsInternal( Position.Start, true, ReadDirection.Backward, - readNext, Array.Empty()); } @@ -204,7 +199,6 @@ protected override async Task ReadAllBackwardsInternal( nextPosition, isEnd, ReadDirection.Backward, - readNext, filteredMessages.ToArray()); } } diff --git a/src/SqlStreamStore.MySql/SqlStreamStore.MySql.csproj b/src/SqlStreamStore.MySql/SqlStreamStore.MySql.csproj index eac91914c..4e3dc2820 100644 --- a/src/SqlStreamStore.MySql/SqlStreamStore.MySql.csproj +++ b/src/SqlStreamStore.MySql/SqlStreamStore.MySql.csproj @@ -3,7 +3,7 @@ João P. Bragança, Damian Hickey MySQL provider for SQL StreamStore Stream Store - MySQL - net5.0 + netstandard2.0 SqlStreamStore.MySql SqlStreamStore.MySql SqlStreamStore diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/AppendToStream.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/AppendToStream.sql similarity index 100% rename from src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/AppendToStream.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV2/AppendToStream.sql diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/DeleteStream.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/DeleteStream.sql similarity index 100% rename from src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/DeleteStream.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV2/DeleteStream.sql diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/DeleteStreamMessages.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/DeleteStreamMessages.sql similarity index 100% rename from src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/DeleteStreamMessages.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV2/DeleteStreamMessages.sql diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/DropAll.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/DropAll.sql similarity index 100% rename from src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/DropAll.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV2/DropAll.sql diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/EnableExplainAnalyze.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/EnableExplainAnalyze.sql similarity index 100% rename from src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/EnableExplainAnalyze.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV2/EnableExplainAnalyze.sql diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/EnforceIdempotentAppend.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/EnforceIdempotentAppend.sql similarity index 100% rename from src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/EnforceIdempotentAppend.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV2/EnforceIdempotentAppend.sql diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ListStreams.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ListStreams.sql similarity index 100% rename from src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ListStreams.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV2/ListStreams.sql diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ListStreamsEndingWith.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ListStreamsEndingWith.sql similarity index 100% rename from src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ListStreamsEndingWith.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV2/ListStreamsEndingWith.sql diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ListStreamsStartingWith.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ListStreamsStartingWith.sql similarity index 100% rename from src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ListStreamsStartingWith.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV2/ListStreamsStartingWith.sql diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Migration.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/Migration.sql similarity index 100% rename from src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Migration.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV2/Migration.sql diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Parameters.cs b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/Parameters.cs similarity index 100% rename from src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Parameters.cs rename to src/SqlStreamStore.Postgres/PgSqlScriptsV2/Parameters.cs diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Read.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/Read.sql similarity index 100% rename from src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Read.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV2/Read.sql diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadAll.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadAll.sql similarity index 100% rename from src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadAll.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadAll.sql diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadHeadPosition.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadHeadPosition.sql similarity index 100% rename from src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadHeadPosition.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadHeadPosition.sql diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadJsonData.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadJsonData.sql similarity index 100% rename from src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadJsonData.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadJsonData.sql diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadSchemaVersion.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadSchemaVersion.sql similarity index 100% rename from src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadSchemaVersion.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadSchemaVersion.sql diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadStreamHeadPosition.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadStreamHeadPosition.sql similarity index 100% rename from src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadStreamHeadPosition.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadStreamHeadPosition.sql diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadStreamHeadVersion.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadStreamHeadVersion.sql similarity index 100% rename from src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadStreamHeadVersion.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadStreamHeadVersion.sql diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadStreamVersionOfMessageId.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadStreamVersionOfMessageId.sql similarity index 100% rename from src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/ReadStreamVersionOfMessageId.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadStreamVersionOfMessageId.sql diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Scavenge.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/Scavenge.sql similarity index 100% rename from src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Scavenge.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV2/Scavenge.sql diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Schema.cs b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/Schema.cs similarity index 100% rename from src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Schema.cs rename to src/SqlStreamStore.Postgres/PgSqlScriptsV2/Schema.cs diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Scripts.cs b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/Scripts.cs similarity index 100% rename from src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Scripts.cs rename to src/SqlStreamStore.Postgres/PgSqlScriptsV2/Scripts.cs diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/SetStreamMetadata.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/SetStreamMetadata.sql similarity index 100% rename from src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/SetStreamMetadata.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV2/SetStreamMetadata.sql diff --git a/src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Tables.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/Tables.sql similarity index 100% rename from src/SqlStreamStore.PostgresV2/PgSqlScriptsV2/Tables.sql rename to src/SqlStreamStore.Postgres/PgSqlScriptsV2/Tables.sql diff --git a/src/SqlStreamStore.Postgres/PostgresStreamStore.Subscriptions.cs b/src/SqlStreamStore.Postgres/PostgresStreamStore.Subscriptions.cs index 481747968..e1d98bb1d 100644 --- a/src/SqlStreamStore.Postgres/PostgresStreamStore.Subscriptions.cs +++ b/src/SqlStreamStore.Postgres/PostgresStreamStore.Subscriptions.cs @@ -2,7 +2,6 @@ { using System; using SqlStreamStore.Infrastructure; - using SqlStreamStore.Streams; using SqlStreamStore.Subscriptions; public partial class PostgresStreamStore diff --git a/src/SqlStreamStore.Postgres/PostgresStreamStore.cs b/src/SqlStreamStore.Postgres/PostgresStreamStore.cs index 0cc50c817..2e2a7abdd 100644 --- a/src/SqlStreamStore.Postgres/PostgresStreamStore.cs +++ b/src/SqlStreamStore.Postgres/PostgresStreamStore.cs @@ -9,13 +9,12 @@ using Npgsql; using SqlStreamStore.Infrastructure; using SqlStreamStore.Logging; - using SqlStreamStore.PgSqlScriptsV1; + using SqlStreamStore.PgSqlScriptsV2; using SqlStreamStore.Subscriptions; /// /// Represents a PostgreSQL stream store implementation. /// - [Obsolete("Use PostgresStreamStoreV2 instead. Note: this will require a schema and data migration.", false)] public partial class PostgresStreamStore : StreamStoreBase { private readonly PostgresStreamStoreSettings _settings; @@ -23,7 +22,7 @@ public partial class PostgresStreamStore : StreamStoreBase private readonly Schema _schema; private readonly Lazy _streamStoreNotifier; - public const int CurrentVersion = 1; + public const int CurrentVersion = 2; /// /// Initializes a new instance of @@ -262,111 +261,69 @@ public string GetSchemaCreationScript() return _schema.Definition; } - //protected override async Task HandleGap( - // PostgresReadAllPage page, - // long fromPositionInclusive, - // int maxCount, - // bool prefetchJsonData, - // CancellationToken cancellationToken) - //{ - // if (page.Messages.Length == 0 || DateTime.UtcNow - page.Messages[page.Messages.Length - 1].CreatedUtc > TimeSpan.FromMinutes(5)) - // return page; - - // // TODO: FIXIT - // // Check for gap between last page and this. - // //if (page.Messages[0].Position != fromPositionInclusive) - // //{ - // // Logger.InfoFormat("Gap detected at lower page boundary. Potentially could have lost {lostMessageCount} events if the gap is transient", page.Messages[0].Position - fromPositionInclusive); - // // page = await HandleGap(page, fromPositionInclusive, maxCount, prefetchJsonData, cancellationToken); - // // //if (!page.IsEnd || page.Messages.Length == 1) - // // // Logger.DebugFormat("Gap detected at lower page boundary. Potentially could have lost {lostMessageCount} events if the gap is transient", page.Messages[0].Position - fromPositionInclusive); - // // //page = await ReloadAfterDelay(fromPositionInclusive, maxCount, prefetchJsonData, ReadNext, cancellationToken); - // //} - - // // check for gap in messages collection - // for (int i = 0; i < page.Messages.Length - 1; i++) - // { - // var expectedNextPosition = page.Messages[i].Position + 1; - // if (expectedNextPosition != page.Messages[i + 1].Position) - // { - // Logger.InfoFormat("Gap detected in " + (page.IsEnd ? "last" : "(NOT the last)") + " page. Returning partial page {fromPosition}-{toPosition}", fromPositionInclusive, fromPositionInclusive + i + 1); - - // PostgresReadAllPage requeryPage; - // var maxPosition = page.Messages[page.Messages.Length - 1].Position; - // do - // { - // requeryPage = await ReadAllForwardsInternal(fromPositionInclusive, maxCount, prefetchJsonData, cancellationToken, maxPosition); - // } while (page.TransactionIds.Intersect(requeryPage.TransactionIds).Any()); - - // return requeryPage; - - // // switched this to return the partial page, then re-issue load starting at gap - // // this speeds up the retry instead of taking a 3 second delay immediately - // //var messagesBeforeGap = new StreamMessage[i+1]; - // //page.Messages.Take(i+1).ToArray().CopyTo(messagesBeforeGap, 0); - // //return new ReadAllPage(page.FromPosition, maxPosition, page.IsEnd, page.Direction, ReadNext, messagesBeforeGap); - // } - // } - - // //ReadAllPage requeryPage; - // //var maxPosition = pageWithGap.Messages[pageWithGap.Messages.Length - 1].Position; - // //do - // //{ - // // requeryPage = await ReadAllForwardsInternal(fromPositionInclusive, maxCount, prefetchJsonData, readNext, cancellationToken, maxPosition); - // //} while (pageWithGap.TxSnapshot.CurrentTxIds.Intersect(requeryPage.TxSnapshot.CurrentTxIds).Any()); - - // return page; - //} - - //protected override async Task HandleGap(T page, long fromPositionInclusive, int maxCount, bool prefetchJsonData, ReadNextAllPage readNext, CancellationToken cancellationToken) where T : PostgresReadAllPage - //{ - // if (page.Messages.Length == 0 || DateTime.UtcNow - page.Messages[page.Messages.Length - 1].CreatedUtc > TimeSpan.FromMinutes(5)) - // return page; - - // // TODO: FIXIT - // // Check for gap between last page and this. - // if (page.Messages[0].Position != fromPositionInclusive) - // { - // Logger.DebugFormat("Gap detected at lower page boundary. Potentially could have lost {lostMessageCount} events if the gap is transient", page.Messages[0].Position - fromPositionInclusive); - // page = await HandleGap(page, fromPositionInclusive, maxCount, prefetchJsonData, readNext, cancellationToken); - // //if (!page.IsEnd || page.Messages.Length == 1) - // // Logger.DebugFormat("Gap detected at lower page boundary. Potentially could have lost {lostMessageCount} events if the gap is transient", page.Messages[0].Position - fromPositionInclusive); - // //page = await ReloadAfterDelay(fromPositionInclusive, maxCount, prefetchJsonData, ReadNext, cancellationToken); - // } - - // // check for gap in messages collection - // for (int i = 0; i < page.Messages.Length - 1; i++) - // { - // var expectedNextPosition = page.Messages[i].Position + 1; - // if (expectedNextPosition != page.Messages[i + 1].Position) - // { - // Logger.InfoFormat("Gap detected in " + (page.IsEnd ? "last" : "(NOT the last)") + " page. Returning partial page {fromPosition}-{toPosition}", fromPositionInclusive, fromPositionInclusive + i + 1); - - // ReadAllPage requeryPage; - // var maxPosition = page.Messages[page.Messages.Length - 1].Position; - // do - // { - // requeryPage = await ReadAllForwardsInternal(fromPositionInclusive, maxCount, prefetchJsonData, readNext, cancellationToken, maxPosition); - // } while (page.TransactionIds.Intersect(requeryPage.TransactionIds).Any()); - - // return requeryPage; - - // // switched this to return the partial page, then re-issue load starting at gap - // // this speeds up the retry instead of taking a 3 second delay immediately - // //var messagesBeforeGap = new StreamMessage[i+1]; - // //page.Messages.Take(i+1).ToArray().CopyTo(messagesBeforeGap, 0); - // //return new ReadAllPage(page.FromPosition, maxPosition, page.IsEnd, page.Direction, ReadNext, messagesBeforeGap); - // } - // } - - // //ReadAllPage requeryPage; - // //var maxPosition = pageWithGap.Messages[pageWithGap.Messages.Length - 1].Position; - // //do - // //{ - // // requeryPage = await ReadAllForwardsInternal(fromPositionInclusive, maxCount, prefetchJsonData, readNext, cancellationToken, maxPosition); - // //} while (pageWithGap.TxSnapshot.CurrentTxIds.Intersect(requeryPage.TxSnapshot.CurrentTxIds).Any()); - - // return page; - //} + /// + /// Returns the script that can be used to migrate to the latest schema version 2. + /// + /// The database creation script. + public string GetMigrationScript() + { + return _schema.Migration; + } + + protected override async Task HandleGap( + PostgresReadAllPage page, + long fromPositionInclusive, + int maxCount, + bool prefetchJsonData, + CancellationToken cancellationToken) + { + if (page.Messages.Length == 0 || DateTime.UtcNow - page.Messages[page.Messages.Length - 1].CreatedUtc > TimeSpan.FromMinutes(5)) + return page; + + // TODO: FIXIT + // Check for gap between last page and this. + //if (page.Messages[0].Position != fromPositionInclusive) + //{ + // Logger.InfoFormat("Gap detected at lower page boundary. Potentially could have lost {lostMessageCount} events if the gap is transient", page.Messages[0].Position - fromPositionInclusive); + // page = await HandleGap(page, fromPositionInclusive, maxCount, prefetchJsonData, cancellationToken); + // //if (!page.IsEnd || page.Messages.Length == 1) + // // Logger.DebugFormat("Gap detected at lower page boundary. Potentially could have lost {lostMessageCount} events if the gap is transient", page.Messages[0].Position - fromPositionInclusive); + // //page = await ReloadAfterDelay(fromPositionInclusive, maxCount, prefetchJsonData, ReadNext, cancellationToken); + //} + + // check for gap in messages collection + for (int i = 0; i < page.Messages.Length - 1; i++) + { + var expectedNextPosition = page.Messages[i].Position + 1; + if (expectedNextPosition != page.Messages[i + 1].Position) + { + Logger.InfoFormat("Gap detected in " + (page.IsEnd ? "last" : "(NOT the last)") + " page. Returning partial page {fromPosition}-{toPosition}", fromPositionInclusive, fromPositionInclusive + i + 1); + + PostgresReadAllPage requeryPage; + var maxPosition = page.Messages[page.Messages.Length - 1].Position; + do + { + requeryPage = await ReadAllForwardsInternal(fromPositionInclusive, maxCount, prefetchJsonData, cancellationToken, maxPosition); + } while (page.TransactionIds.Intersect(requeryPage.TransactionIds).Any()); + + return requeryPage; + + // switched this to return the partial page, then re-issue load starting at gap + // this speeds up the retry instead of taking a 3 second delay immediately + //var messagesBeforeGap = new StreamMessage[i+1]; + //page.Messages.Take(i+1).ToArray().CopyTo(messagesBeforeGap, 0); + //return new ReadAllPage(page.FromPosition, maxPosition, page.IsEnd, page.Direction, ReadNext, messagesBeforeGap); + } + } + + //ReadAllPage requeryPage; + //var maxPosition = pageWithGap.Messages[pageWithGap.Messages.Length - 1].Position; + //do + //{ + // requeryPage = await ReadAllForwardsInternal(fromPositionInclusive, maxCount, prefetchJsonData, readNext, cancellationToken, maxPosition); + //} while (pageWithGap.TxSnapshot.CurrentTxIds.Intersect(requeryPage.TxSnapshot.CurrentTxIds).Any()); + + return page; + } } } diff --git a/src/SqlStreamStore.Postgres/PostgresStreamStoreSettings.cs b/src/SqlStreamStore.Postgres/PostgresStreamStoreSettings.cs index f468994b6..796d93fdd 100644 --- a/src/SqlStreamStore.Postgres/PostgresStreamStoreSettings.cs +++ b/src/SqlStreamStore.Postgres/PostgresStreamStoreSettings.cs @@ -6,7 +6,6 @@ using SqlStreamStore.Infrastructure; using SqlStreamStore.Subscriptions; - [Obsolete("Use PostgresStreamStoreV2Settings instead. Note: this will require a schema and data migration.", false)] public class PostgresStreamStoreSettings { private string _schema = "public"; diff --git a/src/SqlStreamStore.Postgres/SqlStreamStore.Postgres.csproj b/src/SqlStreamStore.Postgres/SqlStreamStore.Postgres.csproj index 732d16a78..164461878 100644 --- a/src/SqlStreamStore.Postgres/SqlStreamStore.Postgres.csproj +++ b/src/SqlStreamStore.Postgres/SqlStreamStore.Postgres.csproj @@ -2,16 +2,16 @@ Postgre SQL provider for SQL StreamStore Stream Store - Postgres - net5.0 + netstandard2.0 SqlStreamStore.Postgres SqlStreamStore.Postgres SqlStreamStore - + - + diff --git a/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Append.cs b/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Append.cs deleted file mode 100644 index 5fab240a8..000000000 --- a/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Append.cs +++ /dev/null @@ -1,91 +0,0 @@ -namespace SqlStreamStore -{ - using System; - using System.Runtime.ExceptionServices; - using System.Threading; - using System.Threading.Tasks; - using Npgsql; - using SqlStreamStore.PgSqlScriptsV2; - using SqlStreamStore.Streams; - - partial class PostgresStreamStoreV2 - { - protected override async Task AppendToStreamInternal( - string streamId, - int expectedVersion, - NewStreamMessage[] messages, - CancellationToken cancellationToken) - { - int maxRetries = 2; //TODO too much? too little? configurable? - Exception exception; - - int retryCount = 0; - do - { - try - { - AppendResult result; - var streamIdInfo = new StreamIdInfo(streamId); - - using(var connection = await OpenConnection(cancellationToken)) - using(var transaction = connection.BeginTransaction()) - using(var command = BuildFunctionCommand( - _schema.AppendToStream, - transaction, - Parameters.StreamId(streamIdInfo.PostgresqlStreamId), - Parameters.StreamIdOriginal(streamIdInfo.PostgresqlStreamId), - Parameters.MetadataStreamId(streamIdInfo.MetadataPosgresqlStreamId), - Parameters.ExpectedVersion(expectedVersion), - Parameters.CreatedUtc(_settings.GetUtcNow?.Invoke()), - Parameters.NewStreamMessages(messages))) - { - try - { - using(var reader = await command - .ExecuteReaderAsync(cancellationToken) - .ConfigureAwait(false)) - { - await reader.ReadAsync(cancellationToken).ConfigureAwait(false); - - result = new AppendResult(reader.GetInt32(0), reader.GetInt64(1)); - } - - await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); - } - catch(PostgresException ex) when(ex.IsWrongExpectedVersion()) - { - await transaction.RollbackAsync(cancellationToken).ConfigureAwait(false); - - throw new WrongExpectedVersionException( - ErrorMessages.AppendFailedWrongExpectedVersion(streamIdInfo.PostgresqlStreamId.IdOriginal, expectedVersion), - streamIdInfo.PostgresqlStreamId.IdOriginal, - expectedVersion, - ex); - } - } - - if(_settings.ScavengeAsynchronously) - { -#pragma warning disable CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed - Task.Run(() => TryScavenge(streamIdInfo, cancellationToken), cancellationToken); -#pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed - } - else - { - await TryScavenge(streamIdInfo, cancellationToken).ConfigureAwait(false); - } - - return result; - } - catch(PostgresException ex) when(ex.IsDeadlock()) - { - exception = ex; - retryCount++; - } - } while(retryCount < maxRetries); - - ExceptionDispatchInfo.Capture(exception).Throw(); - return default; // never actually run - } - } -} diff --git a/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Delete.cs b/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Delete.cs deleted file mode 100644 index 992ea9e77..000000000 --- a/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Delete.cs +++ /dev/null @@ -1,111 +0,0 @@ -namespace SqlStreamStore -{ - using System; - using System.Threading; - using System.Threading.Tasks; - using Npgsql; - using SqlStreamStore.PgSqlScriptsV2; - using SqlStreamStore.Streams; - - public partial class PostgresStreamStoreV2 - { - protected override async Task DeleteStreamInternal( - string streamId, - int expectedVersion, - CancellationToken cancellationToken) - { - var streamIdInfo = new StreamIdInfo(streamId); - - using(var connection = await OpenConnection(cancellationToken)) - using(var transaction = connection.BeginTransaction()) - { - await DeleteStreamInternal( - streamIdInfo.PostgresqlStreamId, - expectedVersion, - transaction, - cancellationToken); - - await DeleteStreamInternal( - streamIdInfo.MetadataPosgresqlStreamId, - ExpectedVersion.Any, - transaction, - cancellationToken); - - await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); - } - } - - private async Task DeleteStreamInternal( - PostgresqlStreamId streamId, - int expectedVersion, - NpgsqlTransaction transaction, - CancellationToken cancellationToken) - { - using(var command = BuildFunctionCommand( - _schema.DeleteStream, - transaction, - Parameters.StreamId(streamId), - Parameters.ExpectedVersion(expectedVersion), - Parameters.CreatedUtc(_settings.GetUtcNow?.Invoke()), - Parameters.DeletionTrackingDisabled(_settings.DisableDeletionTracking), - _settings.DisableDeletionTracking ? Parameters.Empty() : Parameters.DeletedStreamId, - _settings.DisableDeletionTracking ? Parameters.Empty() : Parameters.DeletedStreamIdOriginal, - _settings.DisableDeletionTracking ? Parameters.Empty() : Parameters.DeletedStreamMessage(streamId))) - { - try - { - await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); - } - catch(PostgresException ex) when(ex.IsWrongExpectedVersion()) - { - await transaction.RollbackAsync(cancellationToken).ConfigureAwait(false); - - throw new WrongExpectedVersionException( - ErrorMessages.DeleteStreamFailedWrongExpectedVersion(streamId.IdOriginal, expectedVersion), - streamId.IdOriginal, - expectedVersion, - ex); - } - } - } - - protected override async Task DeleteEventInternal( - string streamId, - Guid eventId, - CancellationToken cancellationToken) - { - var streamIdInfo = new StreamIdInfo(streamId); - - using(var connection = await OpenConnection(cancellationToken)) - using(var transaction = connection.BeginTransaction()) - { - await DeleteEventsInternal(streamIdInfo, new[] { eventId }, transaction, cancellationToken); - - await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); - } - } - - private async Task DeleteEventsInternal( - StreamIdInfo streamIdInfo, - Guid[] eventIds, - NpgsqlTransaction transaction, - CancellationToken cancellationToken) - { - using(var command = BuildFunctionCommand( - _schema.DeleteStreamMessages, - transaction, - Parameters.StreamId(streamIdInfo.PostgresqlStreamId), - Parameters.MessageIds(eventIds), - Parameters.DeletionTrackingDisabled(_settings.DisableDeletionTracking), - _settings.DisableDeletionTracking ? Parameters.Empty() : Parameters.DeletedStreamId, - _settings.DisableDeletionTracking ? Parameters.Empty() : Parameters.DeletedStreamIdOriginal, - Parameters.CreatedUtc(_settings.GetUtcNow?.Invoke()), - _settings.DisableDeletionTracking - ? Parameters.Empty() - : Parameters.DeletedMessages(streamIdInfo.PostgresqlStreamId, eventIds))) - { - await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); - } - } - } -} diff --git a/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.ListStreams.cs b/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.ListStreams.cs deleted file mode 100644 index 25a8cd4b1..000000000 --- a/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.ListStreams.cs +++ /dev/null @@ -1,77 +0,0 @@ -namespace SqlStreamStore -{ - using System.Collections.Generic; - using System.Threading; - using System.Threading.Tasks; - using Npgsql; - using SqlStreamStore.PgSqlScriptsV2; - using SqlStreamStore.Streams; - - public partial class PostgresStreamStoreV2 - { - protected override async Task ListStreamsInternal( - Pattern pattern, - int maxCount, - string continuationToken, - ListNextStreamsPage listNextStreamsPage, - CancellationToken cancellationToken) - { - if(!int.TryParse(continuationToken, out var afterIdInternal)) - { - afterIdInternal = -1; - } - - var streamIds = new List(); - - using(var connection = await OpenConnection(cancellationToken)) - using(var transaction = connection.BeginTransaction()) - using(var command = GetListStreamsCommand(pattern, maxCount, afterIdInternal, transaction)) - using(var reader = await command - .ExecuteReaderAsync(cancellationToken) - .ConfigureAwait(false)) - { - while(await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) - { - streamIds.Add(reader.GetString(0)); - afterIdInternal = reader.GetInt32(1); - } - } - - return new ListStreamsPage(afterIdInternal.ToString(), streamIds.ToArray(), listNextStreamsPage); - } - - private NpgsqlCommand GetListStreamsCommand( - Pattern pattern, - int maxCount, - int? afterIdInternal, - NpgsqlTransaction transaction) - { - switch(pattern) - { - case Pattern.Any _: - return BuildFunctionCommand( - _schema.ListStreams, - transaction, - Parameters.MaxCount(maxCount), - Parameters.OptionalAfterIdInternal(afterIdInternal)); - case Pattern.StartingWith p: - return BuildFunctionCommand( - _schema.ListStreamsStartingWith, - transaction, - Parameters.Pattern(pattern.Value), - Parameters.MaxCount(maxCount), - Parameters.OptionalAfterIdInternal(afterIdInternal)); - case Pattern.EndingWith p: - return BuildFunctionCommand( - _schema.ListStreamsEndingWith, - transaction, - Parameters.Pattern(pattern.Value), - Parameters.MaxCount(maxCount), - Parameters.OptionalAfterIdInternal(afterIdInternal)); - - default: - throw Pattern.Unrecognized(nameof(pattern)); - } - } - } -} diff --git a/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Metadata.cs b/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Metadata.cs deleted file mode 100644 index c0ebba718..000000000 --- a/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Metadata.cs +++ /dev/null @@ -1,101 +0,0 @@ -namespace SqlStreamStore -{ - using System.Threading; - using System.Threading.Tasks; - using Npgsql; - using SqlStreamStore.PgSqlScriptsV2; - using SqlStreamStore.Streams; - - partial class PostgresStreamStoreV2 - { - protected override async Task GetStreamMetadataInternal( - string streamId, - CancellationToken cancellationToken) - { - var streamIdInfo = new StreamIdInfo(streamId); - - using(var connection = await OpenConnection(cancellationToken)) - using(var transaction = connection.BeginTransaction()) - { - return await GetStreamMetadataInternal(streamIdInfo, transaction, cancellationToken); - } - } - - private async Task GetStreamMetadataInternal( - StreamIdInfo streamIdInfo, - NpgsqlTransaction transaction, - CancellationToken cancellationToken) - { - var page = await ReadStreamInternal( - streamIdInfo.MetadataPosgresqlStreamId, - StreamVersion.End, - 1, - ReadDirection.Backward, - true, - null, - transaction, - cancellationToken); - - if(page.Status == PageReadStatus.StreamNotFound) - { - return new StreamMetadataResult(streamIdInfo.PostgresqlStreamId.IdOriginal, -1); - } - - var metadataMessage = await page.Messages[0].GetJsonDataAs(cancellationToken); - - return new StreamMetadataResult( - streamIdInfo.PostgresqlStreamId.IdOriginal, - page.LastStreamVersion, - metadataMessage.MaxAge, - metadataMessage.MaxCount, - metadataMessage.MetaJson); - } - - protected override async Task SetStreamMetadataInternal( - string streamId, - int expectedStreamMetadataVersion, - int? maxAge, - int? maxCount, - string metadataJson, - CancellationToken cancellationToken) - { - int currentVersion; - - var metadata = new MetadataMessage - { - StreamId = streamId, - MaxAge = maxAge, - MaxCount = maxCount, - MetaJson = metadataJson - }; - - var streamIdInfo = new StreamIdInfo(streamId); - - using(var connection = await OpenConnection(cancellationToken)) - using(var transaction = connection.BeginTransaction()) - using(var command = BuildFunctionCommand( - _schema.SetStreamMetadata, - transaction, - Parameters.StreamId(streamIdInfo.PostgresqlStreamId), - Parameters.MetadataStreamId(streamIdInfo.MetadataPosgresqlStreamId), - Parameters.MetadataStreamIdOriginal(streamIdInfo.MetadataPosgresqlStreamId), - Parameters.OptionalMaxAge(metadata.MaxAge), - Parameters.OptionalMaxCount(metadata.MaxCount), - Parameters.ExpectedVersion(expectedStreamMetadataVersion), - Parameters.CreatedUtc(_settings.GetUtcNow?.Invoke()), - Parameters.MetadataStreamMessage( - streamIdInfo.MetadataPosgresqlStreamId, - expectedStreamMetadataVersion, - metadata))) - { - currentVersion = (int) await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false); - - await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); - } - - await TryScavenge(streamIdInfo, cancellationToken); - - return new SetStreamMetadataResult(currentVersion); - } - } -} diff --git a/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Read.cs b/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Read.cs deleted file mode 100644 index fe2aa15f8..000000000 --- a/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Read.cs +++ /dev/null @@ -1,278 +0,0 @@ -namespace SqlStreamStore -{ - using System; - using System.Collections.Generic; - using System.Data; - using System.Data.Common; - using System.Linq; - using System.Text; - using System.Threading; - using System.Threading.Tasks; - using Npgsql; - using SqlStreamStore.PgSqlScriptsV2; - using SqlStreamStore.Streams; - - partial class PostgresStreamStoreV2 - { - private async Task ReadStreamInternal( - PostgresqlStreamId streamId, - int start, - int count, - ReadDirection direction, - bool prefetch, - ReadNextStreamPage readNext, - NpgsqlTransaction transaction, - CancellationToken cancellationToken) - { - // If the count is int.MaxValue, TSql will see it as a negative number. - // Users shouldn't be using int.MaxValue in the first place anyway. - count = count == int.MaxValue ? count - 1 : count; - - // To read backwards from end, need to use int MaxValue - var streamVersion = start == StreamVersion.End ? int.MaxValue : start; - - var messages = new List<(StreamMessage message, int? maxAge)>(); - - Func, int, int> getNextVersion; - - if(direction == ReadDirection.Forward) - { - getNextVersion = (events, lastVersion) => - { - if(events.Any()) - { - return events.Last().StreamVersion + 1; - } - - return lastVersion + 1; - }; - } - else - { - getNextVersion = (events, lastVersion) => - { - if(events.Any()) - { - return events.Last().StreamVersion - 1; - } - - return -1; - }; - } - - var refcursorSql = new StringBuilder(); - - using(var command = BuildFunctionCommand( - _schema.Read, - transaction, - Parameters.StreamId(streamId), - Parameters.Count(count + 1), - Parameters.Version(streamVersion), - Parameters.ReadDirection(direction), - Parameters.Prefetch(prefetch))) - using(var reader = await command - .ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken) - .ConfigureAwait(false)) - { - while(await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) - { - refcursorSql.AppendLine(Schema.FetchAll(reader.GetString(0))); - } - } - - using(var command = new NpgsqlCommand(refcursorSql.ToString(), transaction.Connection, transaction)) - using(var reader = await command - .ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken) - .ConfigureAwait(false)) - { - if(!reader.HasRows) - { - return new ReadStreamPage( - streamId.IdOriginal, - PageReadStatus.StreamNotFound, - start, - -1, - -1, - -1, - direction, - true, - readNext); - } - - if(messages.Count == count) - { - messages.Add(default); - } - - await reader.ReadAsync(cancellationToken).ConfigureAwait(false); - - var lastVersion = reader.GetInt32(0); - var lastPosition = reader.GetInt64(1); - var maxAge = reader.GetFieldValue(2); - - await reader.NextResultAsync(cancellationToken).ConfigureAwait(false); - - while(await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) - { - messages.Add((await ReadStreamMessage(reader, streamId, prefetch), maxAge)); - } - - var isEnd = true; - - if(messages.Count == count + 1) - { - isEnd = false; - messages.RemoveAt(count); - } - - var filteredMessages = FilterExpired(messages); - - return new ReadStreamPage( - streamId.IdOriginal, - PageReadStatus.Success, - start, - getNextVersion(filteredMessages, lastVersion), - lastVersion, - lastPosition, - direction, - isEnd, - readNext, - filteredMessages.ToArray()); - } - } - - protected override async Task ReadStreamForwardsInternal( - string streamId, - int start, - int count, - bool prefetch, - ReadNextStreamPage readNext, - CancellationToken cancellationToken) - { - var streamIdInfo = new StreamIdInfo(streamId); - - using(var connection = await OpenConnection(cancellationToken)) - using(var transaction = connection.BeginTransaction()) - { - return await ReadStreamInternal(streamIdInfo.PostgresqlStreamId, - start, - count, - ReadDirection.Forward, - prefetch, - readNext, - transaction, - cancellationToken); - } - } - - protected override async Task ReadStreamBackwardsInternal( - string streamId, - int fromVersionInclusive, - int count, - bool prefetch, - ReadNextStreamPage readNext, - CancellationToken cancellationToken) - { - var streamIdInfo = new StreamIdInfo(streamId); - - using(var connection = await OpenConnection(cancellationToken)) - using(var transaction = connection.BeginTransaction()) - { - return await ReadStreamInternal(streamIdInfo.PostgresqlStreamId, - fromVersionInclusive, - count, - ReadDirection.Backward, - prefetch, - readNext, - transaction, - cancellationToken); - } - } - - private async Task ReadStreamMessage( - DbDataReader reader, - PostgresqlStreamId streamId, - bool prefetch) - { - async Task ReadString(int ordinal) - { - if(reader.IsDBNull(ordinal)) - { - return null; - } - - using(var textReader = reader.GetTextReader(ordinal)) - { - return await textReader.ReadToEndAsync().ConfigureAwait(false); - } - } - - var messageId = reader.GetGuid(1); - var streamVersion = reader.GetInt32(2); - var position = reader.GetInt64(3); - var createdUtc = reader.GetDateTime(4); - var type = reader.GetString(5); - var jsonMetadata = await ReadString(6); - - if(prefetch) - { - return new StreamMessage( - streamId.IdOriginal, - messageId, - streamVersion, - position, - createdUtc, - type, - jsonMetadata, - await ReadString(7)); - } - - return - new StreamMessage( - streamId.IdOriginal, - messageId, - streamVersion, - position, - createdUtc, - type, - jsonMetadata, - ct => GetJsonData(streamId, streamVersion)(ct)); - } - - protected override async Task ReadHeadPositionInternal(CancellationToken cancellationToken) - { - using(var connection = await OpenConnection(cancellationToken)) - using(var transaction = connection.BeginTransaction()) - using(var command = BuildFunctionCommand(_schema.ReadAllHeadPosition, transaction)) - { - var result = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false); - - return result == DBNull.Value ? Position.End : (long) result; - } - } - - protected override async Task ReadStreamHeadPositionInternal(string streamId, CancellationToken cancellationToken) - { - using(var connection = await OpenConnection(cancellationToken)) - using(var transaction = connection.BeginTransaction()) - using(var command = BuildFunctionCommand(_schema.ReadStreamHeadPosition, transaction, Parameters.StreamId(new PostgresqlStreamId(streamId)))) - { - var result = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false); - - return result == DBNull.Value ? Position.End : (long) result; - } - } - - protected override async Task ReadStreamHeadVersionInternal(string streamId, CancellationToken cancellationToken) - { - using(var connection = await OpenConnection(cancellationToken)) - using(var transaction = connection.BeginTransaction()) - using(var command = BuildFunctionCommand(_schema.ReadStreamHeadVersion, transaction, Parameters.StreamId(new PostgresqlStreamId(streamId)))) - { - var result = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false); - - return result == DBNull.Value ? StreamVersion.End : (int) result; - } - } - } -} diff --git a/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.ReadAll.cs b/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.ReadAll.cs deleted file mode 100644 index c3f081fda..000000000 --- a/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.ReadAll.cs +++ /dev/null @@ -1,212 +0,0 @@ -namespace SqlStreamStore -{ - using System; - using System.Collections.Generic; - using System.Data; - using System.Data.Common; - using System.Threading; - using System.Threading.Tasks; - using SqlStreamStore.PgSqlScriptsV2; - using SqlStreamStore.Streams; - - partial class PostgresStreamStoreV2 - { - protected override async Task ReadAllForwardsInternal( - long fromPositionExclusive, - int maxCount, - bool prefetch, - ReadNextAllPage readNext, - CancellationToken cancellationToken) - { - maxCount = maxCount == int.MaxValue ? maxCount - 1 : maxCount; - - using(var connection = await OpenConnection(cancellationToken)) - using(var transaction = connection.BeginTransaction()) - using(var command = BuildFunctionCommand( - _schema.ReadAll, - transaction, - Parameters.Count(maxCount + 1), - Parameters.Position(fromPositionExclusive), - Parameters.ReadDirection(ReadDirection.Forward), - Parameters.Prefetch(prefetch))) - using(var reader = await command - .ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken) - .ConfigureAwait(false)) - { - if(!reader.HasRows) - { - return new ReadAllPage( - fromPositionExclusive, - fromPositionExclusive, - true, - ReadDirection.Forward, - readNext, - Array.Empty()); - } - - var messages = new List<(StreamMessage message, int? maxAge)>(); - - while(await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) - { - if(messages.Count == maxCount) - { - messages.Add(default); - } - else - { - var streamIdInfo = new StreamIdInfo(reader.GetString(0)); - var (message, maxAge, _) = - await ReadAllStreamMessage(reader, streamIdInfo.PostgresqlStreamId, prefetch); - messages.Add((message, maxAge)); - } - } - - bool isEnd = true; - - if(messages.Count == maxCount + 1) // An extra row was read, we're not at the end - { - isEnd = false; - messages.RemoveAt(maxCount); - } - - var filteredMessages = FilterExpired(messages); - - var nextPosition = filteredMessages[filteredMessages.Count - 1].Position + 1; - - return new ReadAllPage( - fromPositionExclusive, - nextPosition, - isEnd, - ReadDirection.Forward, - readNext, - filteredMessages.ToArray()); - } - } - - protected override async Task ReadAllBackwardsInternal( - long fromPositionExclusive, - int maxCount, - bool prefetch, - ReadNextAllPage readNext, - CancellationToken cancellationToken) - { - maxCount = maxCount == int.MaxValue ? maxCount - 1 : maxCount; - var ordinal = fromPositionExclusive == Position.End ? long.MaxValue : fromPositionExclusive; - - using(var connection = await OpenConnection(cancellationToken)) - using(var transaction = connection.BeginTransaction()) - using(var command = BuildFunctionCommand( - _schema.ReadAll, - transaction, - Parameters.Count(maxCount + 1), - Parameters.Position(ordinal), - Parameters.ReadDirection(ReadDirection.Backward), - Parameters.Prefetch(prefetch))) - using(var reader = await command - .ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken) - .ConfigureAwait(false)) - { - if(!reader.HasRows) - { - // When reading backwards and there are no more items, then next position is LongPosition.Start, - // regardless of what the fromPosition is. - return new ReadAllPage( - Position.Start, - Position.Start, - true, - ReadDirection.Backward, - readNext, - Array.Empty()); - } - - var messages = new List<(StreamMessage message, int? maxAge)>(); - - long lastOrdinal = 0; - while(await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) - { - var streamIdInfo = new StreamIdInfo(reader.GetString(0)); - var (message, maxAge, position) = await ReadAllStreamMessage(reader, streamIdInfo.PostgresqlStreamId, prefetch); - messages.Add((message, maxAge)); - - lastOrdinal = position; - } - - bool isEnd = true; - var nextPosition = lastOrdinal; - - if(messages.Count == maxCount + 1) // An extra row was read, we're not at the end - { - isEnd = false; - messages.RemoveAt(maxCount); - } - - var filteredMessages = FilterExpired(messages); - - fromPositionExclusive = filteredMessages.Count > 0 ? filteredMessages[0].Position : 0; - - return new ReadAllPage( - fromPositionExclusive, - nextPosition, - isEnd, - ReadDirection.Backward, - readNext, - filteredMessages.ToArray()); - } - } - - private async Task<(StreamMessage message, int? maxAge, long position)> ReadAllStreamMessage( - DbDataReader reader, - PostgresqlStreamId streamId, - bool prefetch) - { - async Task ReadString(int ordinal) - { - if(reader.IsDBNull(ordinal)) - { - return null; - } - - using(var textReader = reader.GetTextReader(ordinal)) - { - return await textReader.ReadToEndAsync().ConfigureAwait(false); - } - } - - var messageId = reader.GetGuid(1); - var streamVersion = reader.GetInt32(2); - var position = reader.GetInt64(3); - var createdUtc = reader.GetDateTime(4); - var type = reader.GetString(5); - var jsonMetadata = await ReadString(6); - - if(prefetch) - { - return ( - new StreamMessage( - streamId.IdOriginal, - messageId, - streamVersion, - position, - createdUtc, - type, - jsonMetadata, - await ReadString(7)), - reader.GetFieldValue(8), - position); - } - - return ( - new StreamMessage( - streamId.IdOriginal, - messageId, - streamVersion, - position, - createdUtc, - type, - jsonMetadata, - ct => GetJsonData(streamId, streamVersion)(ct)), - reader.GetFieldValue(8), - position); - } - } -} diff --git a/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Subscriptions.cs b/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Subscriptions.cs deleted file mode 100644 index 854d6aa27..000000000 --- a/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.Subscriptions.cs +++ /dev/null @@ -1,47 +0,0 @@ -namespace SqlStreamStore -{ - using System; - using SqlStreamStore.Infrastructure; - using SqlStreamStore.Subscriptions; - - public partial class PostgresStreamStoreV2 - { - protected override IStreamSubscription SubscribeToStreamInternal( - string streamId, - int? startVersion, - StreamMessageReceived streamMessageReceived, - SubscriptionDropped subscriptionDropped, - HasCaughtUp hasCaughtUp, - bool prefetchJsonData, - string name) => new StreamSubscription( - streamId, - startVersion, - this, - GetStoreObservable, - streamMessageReceived, - subscriptionDropped, - hasCaughtUp, - prefetchJsonData, - name); - - - protected override IAllStreamSubscription SubscribeToAllInternal( - long? fromPosition, - AllStreamMessageReceived streamMessageReceived, - AllSubscriptionDropped subscriptionDropped, - HasCaughtUp hasCaughtUp, - bool prefetchJsonData, - string name) - => new AllStreamSubscription( - fromPosition, - this, - GetStoreObservable, - streamMessageReceived, - subscriptionDropped, - hasCaughtUp, - prefetchJsonData, - name); - - private IObservable GetStoreObservable => _streamStoreNotifier.Value; - } -} \ No newline at end of file diff --git a/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.cs b/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.cs deleted file mode 100644 index d0a3be19a..000000000 --- a/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2.cs +++ /dev/null @@ -1,272 +0,0 @@ -namespace SqlStreamStore -{ - using System; - using System.Collections.Generic; - using System.Data; - using System.Threading; - using System.Threading.Tasks; - using Npgsql; - using SqlStreamStore.Infrastructure; - using SqlStreamStore.Logging; - using SqlStreamStore.PgSqlScriptsV2; - using SqlStreamStore.Subscriptions; - - /// - /// Represents a PostgreSQL stream store implementation. - /// - public partial class PostgresStreamStoreV2 : StreamStoreBase - { - private readonly PostgresStreamStoreV2Settings _settings; - private readonly Func _createConnection; - private readonly Schema _schema; - private readonly Lazy _streamStoreNotifier; - - public const int CurrentVersion = 2; - - /// - /// Initializes a new instance of - /// - /// A settings class to configure this instance. - public PostgresStreamStoreV2(PostgresStreamStoreV2Settings settings) - : base(settings.GetUtcNow, settings.LogName) - { - _settings = settings; - _createConnection = () => _settings.ConnectionFactory(_settings.ConnectionString); - _streamStoreNotifier = new Lazy(() => - { - if(_settings.CreateStreamStoreNotifier == null) - { - throw new InvalidOperationException( - "Cannot create notifier because supplied createStreamStoreNotifier was null"); - } - - return settings.CreateStreamStoreNotifier.Invoke(this); - }); - _schema = new Schema(_settings.Schema); - } - - private async Task OpenConnection(CancellationToken cancellationToken) - { - var connection = _createConnection(); - - await connection.OpenAsync(cancellationToken).ConfigureAwait(false); - - connection.ReloadTypes(); - - connection.TypeMapper.MapComposite(_schema.NewStreamMessage); - - if(_settings.ExplainAnalyze) - { - using(var command = new NpgsqlCommand(_schema.EnableExplainAnalyze, connection)) - { - await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); - } - } - - return connection; - } - - /// - /// Creates a scheme that will hold streams and messages, if the schema does not exist. - /// Calls to this should part of an application's deployment/upgrade process and - /// not every time your application boots up. - /// - /// The cancellation instruction. - /// A representing the asynchronous operation. - public async Task CreateSchemaIfNotExists(CancellationToken cancellationToken = default) - { - using(var connection = _createConnection()) - { - await connection.OpenAsync(cancellationToken).ConfigureAwait(false); - using(var transaction = connection.BeginTransaction()) - { - using(var command = BuildCommand($"CREATE SCHEMA IF NOT EXISTS {_settings.Schema}", transaction)) - { - await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); - } - - using(var command = BuildCommand(_schema.Definition, transaction)) - { - await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); - } - - await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); - } - } - } - - /// - /// Drops all tables related to this store instance. - /// - /// The cancellation instruction. - /// A representing the asynchronous operation. - public async Task DropAll(CancellationToken cancellationToken = default) - { - GuardAgainstDisposed(); - - using(var connection = _createConnection()) - { - await connection.OpenAsync(cancellationToken).ConfigureAwait(false); - using(var transaction = connection.BeginTransaction()) - using(var command = BuildCommand(_schema.DropAll, transaction)) - { - await command - .ExecuteNonQueryAsync(cancellationToken) - .ConfigureAwait(false); - - await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); - } - } - } - - /// - /// Checks the store schema for the correct version. - /// - /// - /// A representing the result of the operation. - public async Task CheckSchema(CancellationToken cancellationToken = default) - { - using(var connection = _createConnection()) - { - await connection.OpenAsync(cancellationToken).ConfigureAwait(false); - using(var transaction = connection.BeginTransaction()) - using(var command = BuildFunctionCommand(_schema.ReadSchemaVersion, transaction)) - { - var result = (int) await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false); - - return new CheckSchemaResult(result, CurrentVersion); - } - } - } - - private Func> GetJsonData(PostgresqlStreamId streamId, int version) - => async cancellationToken => - { - using(var connection = await OpenConnection(cancellationToken)) - using(var transaction = connection.BeginTransaction()) - using(var command = BuildFunctionCommand( - _schema.ReadJsonData, - transaction, - Parameters.StreamId(streamId), - Parameters.Version(version))) - using(var reader = await command - .ExecuteReaderAsync(CommandBehavior.SequentialAccess, cancellationToken) - .ConfigureAwait(false)) - { - if(!await reader.ReadAsync(cancellationToken).ConfigureAwait(false) || reader.IsDBNull(0)) - { - return null; - } - - using(var textReader = reader.GetTextReader(0)) - { - return await textReader.ReadToEndAsync().ConfigureAwait(false); - } - } - }; - - private static NpgsqlCommand BuildFunctionCommand( - string function, - NpgsqlTransaction transaction, - params NpgsqlParameter[] parameters) - { - var command = new NpgsqlCommand(function, transaction.Connection, transaction) - { - CommandType = CommandType.StoredProcedure, - }; - - foreach(var parameter in parameters) - { - command.Parameters.Add(parameter); - } - - return command; - } - - private static NpgsqlCommand BuildCommand( - string commandText, - NpgsqlTransaction transaction) => new NpgsqlCommand(commandText, transaction.Connection, transaction); - - internal async Task TryScavenge( - StreamIdInfo streamIdInfo, - CancellationToken cancellationToken) - { - if(streamIdInfo.PostgresqlStreamId == PostgresqlStreamId.Deleted) - { - return -1; - } - - try - { - using(var connection = await OpenConnection(cancellationToken)) - using(var transaction = connection.BeginTransaction()) - { - var deletedMessageIds = new List(); - using(var command = BuildFunctionCommand( - _schema.Scavenge, - transaction, - Parameters.StreamId(streamIdInfo.PostgresqlStreamId))) - using(var reader = await command - .ExecuteReaderAsync(cancellationToken) - .ConfigureAwait(false)) - { - while(await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) - { - deletedMessageIds.Add(reader.GetGuid(0)); - } - } - - Logger.Info( - "Found {count} message(s) for stream {streamId} to scavenge.", - deletedMessageIds.Count, - streamIdInfo.PostgresqlStreamId); - - if(deletedMessageIds.Count > 0) - { - Logger.Debug( - "Scavenging the following messages on stream {streamId}: {messageIds}", - streamIdInfo.PostgresqlStreamId, - deletedMessageIds); - - await DeleteEventsInternal( - streamIdInfo, - deletedMessageIds.ToArray(), - transaction, - cancellationToken).ConfigureAwait(false); - } - - await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); - - return deletedMessageIds.Count; - } - } - catch(Exception ex) - { - Logger.WarnException( - "Scavenge attempt failed on stream {streamId}. Another attempt will be made when this stream is written to.", - ex, - streamIdInfo.PostgresqlStreamId.IdOriginal); - } - - return -1; - } - - /// - /// Returns the script that can be used to create the Sql Stream Store in a Postgres database. - /// - /// The database creation script. - public string GetSchemaCreationScript() - { - return _schema.Definition; - } - - /// - /// Returns the script that can be used to migrate to the latest schema version 2. - /// - /// The database creation script. - public string GetMigrationScript() - { - return _schema.Migration; - } - } -} diff --git a/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2Settings.cs b/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2Settings.cs deleted file mode 100644 index caea979da..000000000 --- a/src/SqlStreamStore.PostgresV2/PostgresStreamStoreV2Settings.cs +++ /dev/null @@ -1,100 +0,0 @@ -namespace SqlStreamStore -{ - using System; - using Npgsql; - using SqlStreamStore.Imports.Ensure.That; - using SqlStreamStore.Infrastructure; - using SqlStreamStore.Subscriptions; - - public class PostgresStreamStoreV2Settings - { - private string _schema = "public"; - private Func _connectionFactory; - private GetUtcNow _getUtcNow = SystemClock.GetUtcNow; - - /// - /// Initializes a new instance of . - /// - /// The connection string. - public PostgresStreamStoreV2Settings(string connectionString) - { - Ensure.That(connectionString, nameof(connectionString)).IsNotNullOrWhiteSpace(); - - ConnectionString = connectionString; - } - - /// - /// Gets the connection string. - /// - public string ConnectionString { get; } - - /// - /// Allows overriding of the stream store notifier. The default implementation - /// creates . - /// - public CreateStreamStoreNotifier CreateStreamStoreNotifier { get; set; } = - store => new PollingStreamStoreNotifier(store); - - /// - /// The schema SQL Stream Store should place database objects into. Defaults to "public". - /// - public string Schema - { - get => _schema; - set - { - Ensure.That(value, nameof(Schema)).IsNotNullOrWhiteSpace(); - - _schema = value; - } - } - - /// - /// Loads the auto_explain module and turns it on for all queries. Useful for index tuning. - /// - public bool ExplainAnalyze { get; set; } - - /// - /// A delegate to return the current UTC now. Used in testing to - /// control timestamps and time related operations. - /// - public GetUtcNow GetUtcNow - { - get => _getUtcNow; - set => _getUtcNow = value ?? SystemClock.GetUtcNow; - } - - /// - /// The log name used for any of the log messages. - /// - public string LogName { get; } = nameof(PostgresStreamStoreV2); - - /// - /// Allows setting whether or not deleting expired (i.e., older than maxCount) messages happens in the same database transaction as append to stream or not. - /// This does not effect scavenging when setting a stream's metadata - it is always run in the same transaction. - /// - public bool ScavengeAsynchronously { get; set; } = true; - - /// - /// Allows overriding the way a is created given a connection string. - /// The default implementation simply passes the connection string into the constructor. - /// - public Func ConnectionFactory - { - get => _connectionFactory - ?? (_connectionFactory = connectionString => new NpgsqlConnection(connectionString)); - set - { - Ensure.That(value, nameof(value)).IsNotNull(); - _connectionFactory = value; - } - } - - /// - /// Disables stream and message deletion tracking. Will increase - /// performance, however subscribers won't know if a stream or a - /// message has been deleted. This can be modified at runtime. - /// - public bool DisableDeletionTracking { get; set; } - } -} \ No newline at end of file diff --git a/src/SqlStreamStore.PostgresV2/Properties/AssemblyInfo.cs b/src/SqlStreamStore.PostgresV2/Properties/AssemblyInfo.cs deleted file mode 100644 index fbcd872de..000000000 --- a/src/SqlStreamStore.PostgresV2/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,3 +0,0 @@ -using System.Runtime.CompilerServices; - -[assembly: InternalsVisibleTo("SqlStreamStore.Postgres.V2.Tests")] \ No newline at end of file diff --git a/src/SqlStreamStore.PostgresV2/SqlStreamStore.PostgresV2.csproj b/src/SqlStreamStore.PostgresV2/SqlStreamStore.PostgresV2.csproj deleted file mode 100644 index 9f551ec94..000000000 --- a/src/SqlStreamStore.PostgresV2/SqlStreamStore.PostgresV2.csproj +++ /dev/null @@ -1,20 +0,0 @@ - - - Postgre SQL provider for SQL StreamStore - Stream Store - Postgres - netstandard2.0 - SqlStreamStore.PostgresV2 - SqlStreamStore.PostgresV2 - SqlStreamStore - - - - - - - - - - - - \ No newline at end of file diff --git a/src/SqlStreamStore.SchemaCreationScriptTool/SqlStreamStore.SchemaCreationScriptTool.csproj b/src/SqlStreamStore.SchemaCreationScriptTool/SqlStreamStore.SchemaCreationScriptTool.csproj index 5332b6e80..d284c7bba 100644 --- a/src/SqlStreamStore.SchemaCreationScriptTool/SqlStreamStore.SchemaCreationScriptTool.csproj +++ b/src/SqlStreamStore.SchemaCreationScriptTool/SqlStreamStore.SchemaCreationScriptTool.csproj @@ -4,7 +4,6 @@ A tool to generate the schema creation script of the various SQL based implementations. SQL Stream Store - Schema Creation Script Tool Exe - net5.0 net6.0 true SqlStreamStore.SchemaCreationScriptTool diff --git a/src/SqlStreamStore.Sqlite/SqlStreamStore.Sqlite.csproj b/src/SqlStreamStore.Sqlite/SqlStreamStore.Sqlite.csproj index 9dd4c3283..f55d77460 100644 --- a/src/SqlStreamStore.Sqlite/SqlStreamStore.Sqlite.csproj +++ b/src/SqlStreamStore.Sqlite/SqlStreamStore.Sqlite.csproj @@ -2,7 +2,7 @@ Sqlite provider for SQL Stream Store SQL Stream Store - Sqlite Provider - net5.0 + netstandard2.0 true SqlStreamStore.Sqlite SqlStreamStore.Sqlite diff --git a/src/SqlStreamStore.TestUtils/SqlStreamStore.TestUtils.csproj b/src/SqlStreamStore.TestUtils/SqlStreamStore.TestUtils.csproj index 13b27ae06..13180dfe4 100644 --- a/src/SqlStreamStore.TestUtils/SqlStreamStore.TestUtils.csproj +++ b/src/SqlStreamStore.TestUtils/SqlStreamStore.TestUtils.csproj @@ -1,6 +1,5 @@  - net5.0 net6.0 SqlStreamStore.TestUtils false diff --git a/src/SqlStreamStore/SqlStreamStore.csproj b/src/SqlStreamStore/SqlStreamStore.csproj index e7a11c580..25e0c13a3 100644 --- a/src/SqlStreamStore/SqlStreamStore.csproj +++ b/src/SqlStreamStore/SqlStreamStore.csproj @@ -2,7 +2,7 @@ A library for writing and reading streams (also known as an 'event store') with a focus on SQL based implementations. This contains the core interfaces, abstractions and an in-memory implementation for tests. SQL Stream Store - net5.0 + netstandard2.0 $(DefineConstants);LIBLOG_PUBLIC true SqlStreamStore diff --git a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.cs b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.cs index 7978f068c..cb1ab6ea5 100644 --- a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.cs +++ b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.cs @@ -11,7 +11,7 @@ using Xunit; using Xunit.Abstractions; - public abstract partial class AcceptanceTests : IAsyncLifetime + public abstract partial class AcceptanceTests : IAsyncLifetime where TReadAllPage : IReadAllPage { private const string DefaultJsonData = @"{ ""data"": ""data"" }"; private const string DefaultJsonMetadata = @"{ ""meta"": ""data"" }"; @@ -28,9 +28,9 @@ public async Task InitializeAsync() Fixture = await CreateFixture(); } - private IStreamStore Store => Fixture.Store; + private IStreamStore Store => Fixture.Store; - protected IStreamStoreFixture Fixture { get; private set; } + protected IStreamStoreFixture Fixture { get; private set; } protected ITestOutputHelper TestOutputHelper { get; } @@ -41,7 +41,7 @@ public Task DisposeAsync() return Task.CompletedTask; } - protected abstract Task CreateFixture(); + protected abstract Task> CreateFixture(); private static IDisposable CaptureLogs(ITestOutputHelper testOutputHelper) => LoggingHelper.Capture(testOutputHelper); diff --git a/tests/SqlStreamStore.AcceptanceTests/IStreamStoreFixture.cs b/tests/SqlStreamStore.AcceptanceTests/IStreamStoreFixture.cs index 8b127d197..037a953c1 100644 --- a/tests/SqlStreamStore.AcceptanceTests/IStreamStoreFixture.cs +++ b/tests/SqlStreamStore.AcceptanceTests/IStreamStoreFixture.cs @@ -2,10 +2,11 @@ namespace SqlStreamStore { using System; using SqlStreamStore.Infrastructure; + using SqlStreamStore.Streams; - public interface IStreamStoreFixture: IDisposable + public interface IStreamStoreFixture : IDisposable where TReadPage : IReadAllPage { - IStreamStore Store { get; } + IStreamStore Store { get; } GetUtcNow GetUtcNow { get; set; } diff --git a/tests/SqlStreamStore.HAL.Tests/SqlStreamStore.HAL.Tests.csproj b/tests/SqlStreamStore.HAL.Tests/SqlStreamStore.HAL.Tests.csproj index 21abb99fd..bc504238d 100644 --- a/tests/SqlStreamStore.HAL.Tests/SqlStreamStore.HAL.Tests.csproj +++ b/tests/SqlStreamStore.HAL.Tests/SqlStreamStore.HAL.Tests.csproj @@ -1,6 +1,5 @@  - netcoreapp3.1 net6.0 SqlStreamStore.HAL.Tests SqlStreamStore.HAL.Tests diff --git a/tests/SqlStreamStore.Http.Tests/SqlStreamStore.Http.Tests.csproj b/tests/SqlStreamStore.Http.Tests/SqlStreamStore.Http.Tests.csproj index 25934023f..e7b5343c9 100644 --- a/tests/SqlStreamStore.Http.Tests/SqlStreamStore.Http.Tests.csproj +++ b/tests/SqlStreamStore.Http.Tests/SqlStreamStore.Http.Tests.csproj @@ -1,7 +1,6 @@  - net5.0 net6.0 SqlStreamStore.Http.Tests SqlStreamStore.Http.Tests diff --git a/tests/SqlStreamStore.MsSql.Tests/MsSqlStreamStoreAcceptanceTests.cs b/tests/SqlStreamStore.MsSql.Tests/MsSqlStreamStoreAcceptanceTests.cs index b87181281..fa415bc4a 100644 --- a/tests/SqlStreamStore.MsSql.Tests/MsSqlStreamStoreAcceptanceTests.cs +++ b/tests/SqlStreamStore.MsSql.Tests/MsSqlStreamStoreAcceptanceTests.cs @@ -1,10 +1,11 @@ namespace SqlStreamStore { using System.Threading.Tasks; + using SqlStreamStore.Streams; using Xunit; using Xunit.Abstractions; - public class MsSqlStreamStoreAcceptanceTests : AcceptanceTests, IClassFixture + public class MsSqlStreamStoreAcceptanceTests : AcceptanceTests, IClassFixture { private readonly MsSqlStreamStoreFixturePool _fixturePool; @@ -14,7 +15,7 @@ public MsSqlStreamStoreAcceptanceTests(MsSqlStreamStoreFixturePool fixturePool, _fixturePool = fixturePool; } - protected override async Task CreateFixture() + protected override async Task> CreateFixture() => await _fixturePool.Get(TestOutputHelper); /* diff --git a/tests/SqlStreamStore.MsSql.Tests/MsSqlStreamStoreFixture.cs b/tests/SqlStreamStore.MsSql.Tests/MsSqlStreamStoreFixture.cs index 8faa60f33..4754f35b0 100644 --- a/tests/SqlStreamStore.MsSql.Tests/MsSqlStreamStoreFixture.cs +++ b/tests/SqlStreamStore.MsSql.Tests/MsSqlStreamStoreFixture.cs @@ -3,9 +3,10 @@ namespace SqlStreamStore using System; using System.Threading.Tasks; using SqlStreamStore.Infrastructure; + using SqlStreamStore.Streams; using SqlStreamStore.TestUtils.MsSql; - public class MsSqlStreamStoreFixture : IStreamStoreFixture + public class MsSqlStreamStoreFixture : IStreamStoreFixture { private readonly Action _onDispose; private readonly MsSqlStreamStoreSettings _settings; @@ -33,7 +34,7 @@ public MsSqlStreamStoreFixture( public string DatabaseName { get; } - public IStreamStore Store => MsSqlStreamStore; + public IStreamStore Store => MsSqlStreamStore; public MsSqlStreamStore MsSqlStreamStore { get; private set; } diff --git a/tests/SqlStreamStore.MsSql.Tests/SqlStreamStore.MsSql.Tests.csproj b/tests/SqlStreamStore.MsSql.Tests/SqlStreamStore.MsSql.Tests.csproj index 9de621303..236973360 100644 --- a/tests/SqlStreamStore.MsSql.Tests/SqlStreamStore.MsSql.Tests.csproj +++ b/tests/SqlStreamStore.MsSql.Tests/SqlStreamStore.MsSql.Tests.csproj @@ -1,6 +1,5 @@  - net5.0 net6.0 SqlStreamStore.MsSql.Tests SqlStreamStore.MsSql.Tests diff --git a/tests/SqlStreamStore.MsSql.V3.Tests/MsSqlStreamStoreV3AcceptanceTests.cs b/tests/SqlStreamStore.MsSql.V3.Tests/MsSqlStreamStoreV3AcceptanceTests.cs index 6f9a19bd5..76b46ebab 100644 --- a/tests/SqlStreamStore.MsSql.V3.Tests/MsSqlStreamStoreV3AcceptanceTests.cs +++ b/tests/SqlStreamStore.MsSql.V3.Tests/MsSqlStreamStoreV3AcceptanceTests.cs @@ -9,7 +9,7 @@ using Xunit; using Xunit.Abstractions; - public class MsSqlStreamStoreV3AcceptanceTests : AcceptanceTests, IClassFixture + public class MsSqlStreamStoreV3AcceptanceTests : AcceptanceTests, IClassFixture { private readonly MsSqlStreamStoreV3FixturePool _fixturePool; @@ -19,7 +19,7 @@ public MsSqlStreamStoreV3AcceptanceTests(MsSqlStreamStoreV3FixturePool fixturePo _fixturePool = fixturePool; } - protected override async Task CreateFixture() + protected override async Task> CreateFixture() => await _fixturePool.Get(TestOutputHelper); [Fact] diff --git a/tests/SqlStreamStore.MsSql.V3.Tests/MsSqlStreamStoreV3Fixture.cs b/tests/SqlStreamStore.MsSql.V3.Tests/MsSqlStreamStoreV3Fixture.cs index bc6d8f7b3..adb993e2b 100644 --- a/tests/SqlStreamStore.MsSql.V3.Tests/MsSqlStreamStoreV3Fixture.cs +++ b/tests/SqlStreamStore.MsSql.V3.Tests/MsSqlStreamStoreV3Fixture.cs @@ -5,9 +5,10 @@ using System.Threading.Tasks; using Microsoft.Data.SqlClient; using SqlStreamStore.Infrastructure; + using SqlStreamStore.Streams; using SqlStreamStore.TestUtils.MsSql; - public class MsSqlStreamStoreV3Fixture : IStreamStoreFixture + public class MsSqlStreamStoreV3Fixture : IStreamStoreFixture { private readonly Action _onDispose; private readonly MsSqlStreamStoreV3Settings _settings; @@ -83,7 +84,7 @@ await command public string DatabaseName { get; } - public IStreamStore Store => MsSqlStreamStoreV3; + public IStreamStore Store => MsSqlStreamStoreV3; public MsSqlStreamStoreV3 MsSqlStreamStoreV3 { get; private set; } diff --git a/tests/SqlStreamStore.MsSql.V3.Tests/SqlStreamStore.MsSql.V3.Tests.csproj b/tests/SqlStreamStore.MsSql.V3.Tests/SqlStreamStore.MsSql.V3.Tests.csproj index 9616b7d26..234500363 100644 --- a/tests/SqlStreamStore.MsSql.V3.Tests/SqlStreamStore.MsSql.V3.Tests.csproj +++ b/tests/SqlStreamStore.MsSql.V3.Tests/SqlStreamStore.MsSql.V3.Tests.csproj @@ -1,6 +1,5 @@  - net5.0 net6.0 SqlStreamStore.MsSql.V3.Tests SqlStreamStore diff --git a/tests/SqlStreamStore.MySql.Tests/SqlStreamStore.MySql.Tests.csproj b/tests/SqlStreamStore.MySql.Tests/SqlStreamStore.MySql.Tests.csproj index 99f873046..657a35ece 100644 --- a/tests/SqlStreamStore.MySql.Tests/SqlStreamStore.MySql.Tests.csproj +++ b/tests/SqlStreamStore.MySql.Tests/SqlStreamStore.MySql.Tests.csproj @@ -1,6 +1,5 @@  - net5.0 net6.0 SqlStreamStore.MySql.Tests SqlStreamStore diff --git a/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreAcceptanceTests.cs b/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreAcceptanceTests.cs index e09700c25..78982c664 100644 --- a/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreAcceptanceTests.cs +++ b/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreAcceptanceTests.cs @@ -1,10 +1,11 @@ namespace SqlStreamStore { using System.Threading.Tasks; + using SqlStreamStore.Streams; using Xunit; using Xunit.Abstractions; - public class PostgresStreamStoreAcceptanceTests : AcceptanceTests, IClassFixture + public class PostgresStreamStoreAcceptanceTests : AcceptanceTests, IClassFixture { private readonly PostgresStreamStoreV3FixturePool _fixturePool; @@ -16,7 +17,7 @@ public PostgresStreamStoreAcceptanceTests( _fixturePool = fixturePool; } - protected override async Task CreateFixture() + protected override async Task> CreateFixture() => await _fixturePool.Get(TestOutputHelper); /* diff --git a/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreFixture.cs b/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreFixture.cs index 16ad7f62a..2cab06b52 100644 --- a/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreFixture.cs +++ b/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreFixture.cs @@ -6,7 +6,7 @@ namespace SqlStreamStore using SqlStreamStore.Infrastructure; using SqlStreamStore.TestUtils.Postgres; - public class PostgresStreamStoreFixture : IStreamStoreFixture + public class PostgresStreamStoreFixture : IStreamStoreFixture { private readonly Action _onDispose; private bool _preparedPreviously; diff --git a/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreFixturePool.cs b/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreFixturePool.cs index bd5ede374..74a706329 100644 --- a/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreFixturePool.cs +++ b/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreFixturePool.cs @@ -31,7 +31,7 @@ public async Task Get( schema, dockerInstance, databaseName, - onDispose:() => fixturePool.Enqueue(fixture)); + () => fixturePool.Enqueue(fixture)); outputHelper.WriteLine($"Using new fixture with db {databaseName}"); } diff --git a/tests/SqlStreamStore.Postgres.Tests/SqlStreamStore.Postgres.Tests.csproj b/tests/SqlStreamStore.Postgres.Tests/SqlStreamStore.Postgres.Tests.csproj index 0a2b36a6c..52a732a77 100644 --- a/tests/SqlStreamStore.Postgres.Tests/SqlStreamStore.Postgres.Tests.csproj +++ b/tests/SqlStreamStore.Postgres.Tests/SqlStreamStore.Postgres.Tests.csproj @@ -1,7 +1,6 @@  - net5.0 net6.0 SqlStreamStore.Postgres.Tests SqlStreamStore diff --git a/tests/SqlStreamStore.Postgres.V2.Tests/PostgresStreamStoreAcceptanceTests.cs b/tests/SqlStreamStore.Postgres.V2.Tests/PostgresStreamStoreAcceptanceTests.cs deleted file mode 100644 index 784bee6af..000000000 --- a/tests/SqlStreamStore.Postgres.V2.Tests/PostgresStreamStoreAcceptanceTests.cs +++ /dev/null @@ -1,157 +0,0 @@ -namespace SqlStreamStore -{ - using System.Threading.Tasks; - using Shouldly; - using SqlStreamStore.PgSqlScriptsV2; - using Xunit; - using Xunit.Abstractions; - - public class PostgresStreamStoreAcceptanceTests : AcceptanceTests, IClassFixture - { - private readonly PostgresStreamStoreV3FixturePool _fixturePool; - - public PostgresStreamStoreAcceptanceTests( - PostgresStreamStoreV3FixturePool fixturePool, - ITestOutputHelper testOutputHelper) - : base(testOutputHelper) - { - _fixturePool = fixturePool; - } - - protected override async Task CreateFixture() - => await _fixturePool.Get(TestOutputHelper); - - [Fact] - public async Task Can_check_schema() - { - using(var fixture = await _fixturePool.Get(TestOutputHelper)) - { - var result = await fixture.PostgresStreamStore.CheckSchema(); - - result.ShouldBe(new CheckSchemaResult(2, 2)); - result.IsMatch.ShouldBeTrue(); - } - } - - [Fact] - public async Task Can_export_database_migration_script() - { - var schema = "custom_schema"; - using(var fixture = await _fixturePool.Get(TestOutputHelper, schema)) - { - var sqlScript = fixture.PostgresStreamStore.GetMigrationScript(); - sqlScript.ShouldBe(new Scripts(schema).Migration); - } - } - - /* - [Fact] - public async Task Can_use_multiple_schemas() - { - using (var dboFixture = await CreateFixture()) - { - var dboStore = dboFixture.Store; - - using(var barFixture = await CreateFixture()) - { - var barStore = barFixture.Store; - - await dboStore.AppendToStream("stream-1", - ExpectedVersion.NoStream, - CreateNewStreamMessages(1, 2)); - await barStore.AppendToStream("stream-1", - ExpectedVersion.NoStream, - CreateNewStreamMessages(1, 2)); - - var dboHeadPosition = await dboStore.ReadHeadPosition(); - var barHeadPosition = await barStore.ReadHeadPosition(); - - dboHeadPosition.ShouldBe(1); - barHeadPosition.ShouldBe(1); - } - } - } - - [Fact] - public async Task when_try_scavenge_fails_returns_negative_one() - { - using (var fixture = await PostgresStreamStoreFixture.Create(testOutputHelper: TestOutputHelper)) - { - var cts = new CancellationTokenSource(); - - cts.Cancel(); - - var result = await fixture.Store.TryScavenge(new StreamIdInfo("stream-1"), cts.Token); - - result.ShouldBe(-1); - } - } - - [Theory, InlineData("dbo"), InlineData("myschema")] - public async Task Can_call_initialize_repeatably(string schema) - { - using(var fixture = await PostgresStreamStoreFixture.Create(schema, testOutputHelper: TestOutputHelper, createSchema: false)) - { - await fixture.Store.CreateSchemaIfNotExists(); - await fixture.Store.CreateSchemaIfNotExists(); - } - } - - [Fact] - public async Task Can_drop_all() - { - var streamStoreObjects = new List(); - - string ReadInformationSchema((string name, string table) _) - => $"SELECT {_.name}_name FROM information_schema.{_.table} WHERE {_.name}_schema = 'dbo'"; - - using(var fixture = await PostgresStreamStoreFixture.Create(testOutputHelper: TestOutputHelper)) - { - await fixture.Store.DropAll(); - - var commandText = string.Join( - $"{Environment.NewLine}UNION{Environment.NewLine}", - new[] - { - ("table", "tables"), - ("sequence", "sequences"), - ("constraint", "table_constraints"), - ("user_defined_type", "user_defined_types"), - ("routine", "routines") - }.Select(ReadInformationSchema)); - - using(var connection = new NpgsqlConnection(fixture.ConnectionString)) - { - await connection.OpenAsync().ConfigureAwait(false); - - using(var command = new NpgsqlCommand(commandText, connection)) - using(var reader = await command.ExecuteReaderAsync().ConfigureAwait(false)) - { - while(await reader.ReadAsync().ConfigureAwait(false)) - { - streamStoreObjects.Add(reader.GetString(0)); - } - } - } - - streamStoreObjects.ShouldBeEmpty(); - } - } - - - [Fact] - public void Can_export_database_creation_script() - { - string schema = "custom_schema"; - var store = new PostgresStreamStore(new PostgresStreamStoreSettings("server=.;database=sss") - { - Schema = schema, - }); - - var sqlScript = store.GetSchemaCreationScript(); - sqlScript.ShouldBe(new Scripts(schema).CreateSchema); - } - */ - - } -} diff --git a/tests/SqlStreamStore.Postgres.V2.Tests/PostgresStreamStoreFixture.cs b/tests/SqlStreamStore.Postgres.V2.Tests/PostgresStreamStoreFixture.cs deleted file mode 100644 index a1b8193e4..000000000 --- a/tests/SqlStreamStore.Postgres.V2.Tests/PostgresStreamStoreFixture.cs +++ /dev/null @@ -1,103 +0,0 @@ -namespace SqlStreamStore -{ - using System; - using System.Threading.Tasks; - using Npgsql; - using SqlStreamStore.Infrastructure; - using SqlStreamStore.TestUtils.Postgres; - - public class PostgresStreamStoreFixture : IStreamStoreFixture - { - private readonly Action _onDispose; - private bool _preparedPreviously; - private readonly PostgresStreamStoreV2Settings _settings; - - public PostgresStreamStoreFixture( - string schema, - PostgresContainer dockerInstance, - string databaseName, - Action onDispose) - { - _onDispose = onDispose; - - DatabaseName = databaseName; - var connectionString = dockerInstance.ConnectionString; - - _settings = new PostgresStreamStoreV2Settings(connectionString) - { - Schema = schema, - GetUtcNow = () => GetUtcNow(), - DisableDeletionTracking = false, - ScavengeAsynchronously = false, - }; - } - - public void Dispose() - { - Store.Dispose(); - PostgresStreamStore = null; - _onDispose(); - } - - public string DatabaseName { get; } - - public IStreamStore Store => PostgresStreamStore; - - public PostgresStreamStoreV2 PostgresStreamStore { get; private set; } - - public GetUtcNow GetUtcNow { get; set; } = SystemClock.GetUtcNow; - - public long MinPosition { get; set; } = 0; - - public int MaxSubscriptionCount { get; set; } = 100; - - public bool DisableDeletionTracking - { - get => _settings.DisableDeletionTracking; - set => _settings.DisableDeletionTracking = value; - } - - public async Task Prepare() - { - _settings.DisableDeletionTracking = false; - PostgresStreamStore = new PostgresStreamStoreV2(_settings); - - await PostgresStreamStore.CreateSchemaIfNotExists(); - if (_preparedPreviously) - { - using (var connection = new NpgsqlConnection(_settings.ConnectionString)) - { - connection.Open(); - - var schema = _settings.Schema; - - var commandText = $"DELETE FROM {schema}.messages"; - using (var command = new NpgsqlCommand(commandText, connection)) - { - command.ExecuteNonQuery(); - } - - commandText = $"DELETE FROM {schema}.streams"; - using (var command = new NpgsqlCommand(commandText, connection)) - { - command.ExecuteNonQuery(); - } - - commandText = $"ALTER SEQUENCE {schema}.streams_seq RESTART WITH 1;"; - using (var command = new NpgsqlCommand(commandText, connection)) - { - command.ExecuteNonQuery(); - } - - commandText = $"ALTER SEQUENCE {schema}.messages_seq RESTART WITH 0;"; - using (var command = new NpgsqlCommand(commandText, connection)) - { - command.ExecuteNonQuery(); - } - } - } - - _preparedPreviously = true; - } - } -} \ No newline at end of file diff --git a/tests/SqlStreamStore.Postgres.V2.Tests/PostgresStreamStoreFixturePool.cs b/tests/SqlStreamStore.Postgres.V2.Tests/PostgresStreamStoreFixturePool.cs deleted file mode 100644 index ba5ed6f85..000000000 --- a/tests/SqlStreamStore.Postgres.V2.Tests/PostgresStreamStoreFixturePool.cs +++ /dev/null @@ -1,58 +0,0 @@ -namespace SqlStreamStore -{ - using System; - using System.Collections.Concurrent; - using System.Threading.Tasks; - using SqlStreamStore.TestUtils.Postgres; - using Xunit; - using Xunit.Abstractions; - - public class PostgresStreamStoreV3FixturePool : IAsyncLifetime - { - private readonly ConcurrentDictionary> _fixturePoolBySchema - = new ConcurrentDictionary>(); - - public async Task Get( - ITestOutputHelper outputHelper, - string schema = "dbo") - { - var fixturePool = _fixturePoolBySchema.GetOrAdd( - schema, - _ => new ConcurrentQueue()); - - if (!fixturePool.TryDequeue(out var fixture)) - { - var databaseName = $"test_{Guid.NewGuid():n}"; - var dockerInstance = new PostgresContainer(databaseName); - await dockerInstance.Start(); - await dockerInstance.CreateDatabase(); - - fixture = new PostgresStreamStoreFixture( - schema, - dockerInstance, - databaseName, - onDispose:() => fixturePool.Enqueue(fixture)); - - outputHelper.WriteLine($"Using new fixture with db {databaseName}"); - } - else - { - outputHelper.WriteLine($"Using pooled fixture with db {fixture.DatabaseName}"); - } - - await fixture.Prepare(); - - return fixture; - } - - public Task InitializeAsync() - { - return Task.CompletedTask; - } - - public Task DisposeAsync() - { - return Task.CompletedTask; - } - } -} \ No newline at end of file diff --git a/tests/SqlStreamStore.Postgres.V2.Tests/SqlStreamStore.Postgres.V2.Tests.csproj b/tests/SqlStreamStore.Postgres.V2.Tests/SqlStreamStore.Postgres.V2.Tests.csproj deleted file mode 100644 index ab874c70c..000000000 --- a/tests/SqlStreamStore.Postgres.V2.Tests/SqlStreamStore.Postgres.V2.Tests.csproj +++ /dev/null @@ -1,16 +0,0 @@ - - - - net6.0 - SqlStreamStore.Postgres.V2.Tests - SqlStreamStore - - - - - - - - - - \ No newline at end of file diff --git a/tests/SqlStreamStore.Sqlite.Tests/SqlStreamStore.Sqlite.Tests.csproj b/tests/SqlStreamStore.Sqlite.Tests/SqlStreamStore.Sqlite.Tests.csproj index 92c2cc9cf..185025726 100644 --- a/tests/SqlStreamStore.Sqlite.Tests/SqlStreamStore.Sqlite.Tests.csproj +++ b/tests/SqlStreamStore.Sqlite.Tests/SqlStreamStore.Sqlite.Tests.csproj @@ -1,9 +1,8 @@ - net5.0 SqlStreamStore.Sqlite.Tests net6.0 - SqlStreamStore.Sqlite.Tests + SqlStreamStore.Sqlite.Tests SqlStreamStore From ffafbf673d4b7d3417140b626ae4108c52c2fc68 Mon Sep 17 00:00:00 2001 From: Arne Schoonvliet Date: Wed, 20 Apr 2022 15:37:45 +0200 Subject: [PATCH 12/13] Make tests work --- .../AcceptanceTests.AppendStream.cs | 2 +- .../AcceptanceTests.DeleteEvent.cs | 2 +- .../AcceptanceTests.DeleteStream.cs | 2 +- .../AcceptanceTests.DeletionTracking.cs | 2 +- .../AcceptanceTests.ListStreams.cs | 2 +- .../SqlStreamStore.AcceptanceTests/AcceptanceTests.ReadAll.cs | 2 +- .../AcceptanceTests.ReadHeadCheckpoint.cs | 2 +- .../AcceptanceTests.ReadStream.cs | 2 +- .../AcceptanceTests.StreamLimits.cs | 2 +- .../AcceptanceTests.StreamMetadata.cs | 2 +- .../AcceptanceTests.Subscriptions.cs | 2 +- tests/SqlStreamStore.AcceptanceTests/IStreamStoreFixture.cs | 2 +- tests/SqlStreamStore.MsSql.Tests/ExploratoryTests.cs | 2 +- .../StreamStoreAcceptanceTests.ReadLongStreamInPages.cs | 2 +- tests/SqlStreamStore.MsSql.V3.Tests/ExploratoryTests.cs | 2 +- tests/SqlStreamStore.MsSql.V3.Tests/MigrationTests.cs | 4 ++-- .../StreamStoreAcceptanceTests.ReadLongStreamInPages.cs | 2 +- .../PostgresStreamStoreAcceptanceTests.cs | 1 - 18 files changed, 18 insertions(+), 19 deletions(-) diff --git a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.AppendStream.cs b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.AppendStream.cs index 49b41f47c..4eeb35f65 100644 --- a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.AppendStream.cs +++ b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.AppendStream.cs @@ -7,7 +7,7 @@ using SqlStreamStore.Streams; using Xunit; - public partial class AcceptanceTests + public partial class AcceptanceTests { [Fact, Trait("Category", "AppendStream")] public async Task diff --git a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.DeleteEvent.cs b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.DeleteEvent.cs index 49b10f92f..09e6c1d45 100644 --- a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.DeleteEvent.cs +++ b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.DeleteEvent.cs @@ -8,7 +8,7 @@ using Xunit; using static Streams.Deleted; - public partial class AcceptanceTests + public partial class AcceptanceTests { [Fact, Trait("Category", "DeleteEvent")] public async Task When_delete_message_then_message_should_be_removed_from_stream() diff --git a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.DeleteStream.cs b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.DeleteStream.cs index 4c5a1b549..bc13c7679 100644 --- a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.DeleteStream.cs +++ b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.DeleteStream.cs @@ -8,7 +8,7 @@ using Xunit; using static Streams.Deleted; - public partial class AcceptanceTests + public partial class AcceptanceTests { [Fact, Trait("Category", "DeleteStream")] public async Task When_delete_stream_with_no_expected_version_and_read_then_should_get_StreamNotFound() diff --git a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.DeletionTracking.cs b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.DeletionTracking.cs index 852ecafcd..20ad86899 100644 --- a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.DeletionTracking.cs +++ b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.DeletionTracking.cs @@ -6,7 +6,7 @@ namespace SqlStreamStore using Xunit; using static Streams.Deleted; - partial class AcceptanceTests + partial class AcceptanceTests { [Fact] public async Task When_deletion_tracking_is_disabled_deleted_message_should_not_be_tracked() diff --git a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ListStreams.cs b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ListStreams.cs index c84f1fed6..2945c4599 100644 --- a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ListStreams.cs +++ b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ListStreams.cs @@ -7,7 +7,7 @@ namespace SqlStreamStore using SqlStreamStore.Streams; using Xunit; - public partial class AcceptanceTests + public partial class AcceptanceTests { [Theory] [InlineData("stream")] diff --git a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ReadAll.cs b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ReadAll.cs index 16b6b680f..79f6c0983 100644 --- a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ReadAll.cs +++ b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ReadAll.cs @@ -8,7 +8,7 @@ using SqlStreamStore.Streams; using Xunit; - public partial class AcceptanceTests + public partial class AcceptanceTests { [Fact, Trait("Category", "ReadAll")] public async Task Can_read_all_forwards() diff --git a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ReadHeadCheckpoint.cs b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ReadHeadCheckpoint.cs index 5481de59e..dd08440fc 100644 --- a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ReadHeadCheckpoint.cs +++ b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ReadHeadCheckpoint.cs @@ -5,7 +5,7 @@ using SqlStreamStore.Streams; using Xunit; - public partial class AcceptanceTests + public partial class AcceptanceTests { [Fact] public async Task Given_empty_store_when_get_head_position_Then_should_be_minus_one() diff --git a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ReadStream.cs b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ReadStream.cs index a79b5ae29..cc55114f1 100644 --- a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ReadStream.cs +++ b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.ReadStream.cs @@ -10,7 +10,7 @@ using SqlStreamStore.Streams; using Xunit; - public partial class AcceptanceTests + public partial class AcceptanceTests { [Theory, Trait("Category", "ReadStream")] [MemberData(nameof(GetReadStreamForwardsTheories))] diff --git a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.StreamLimits.cs b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.StreamLimits.cs index 03663e64e..7e787f700 100644 --- a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.StreamLimits.cs +++ b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.StreamLimits.cs @@ -7,7 +7,7 @@ namespace SqlStreamStore using SqlStreamStore.Streams; using Xunit; - public partial class AcceptanceTests + public partial class AcceptanceTests { [Theory, Trait("Category", "StreamMetadata"), InlineData(ExpectedVersion.NoStream), InlineData(ExpectedVersion.Any)] diff --git a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.StreamMetadata.cs b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.StreamMetadata.cs index 64999f5b0..fa1d68685 100644 --- a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.StreamMetadata.cs +++ b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.StreamMetadata.cs @@ -7,7 +7,7 @@ using SqlStreamStore.Streams; using Xunit; - public partial class AcceptanceTests + public partial class AcceptanceTests { private const string DefaultStreamMetadataJson = @"{ ""meta"": ""meta"" }"; //TODO: Port some of the tests from AppendStream with regard to expected version to verify behavior of Get/SetStreamMetadata. diff --git a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.Subscriptions.cs b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.Subscriptions.cs index 4f31d433d..34fdba683 100644 --- a/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.Subscriptions.cs +++ b/tests/SqlStreamStore.AcceptanceTests/AcceptanceTests.Subscriptions.cs @@ -12,7 +12,7 @@ using SqlStreamStore.TestUtils; using Xunit; - public partial class AcceptanceTests + public partial class AcceptanceTests { [Fact, Trait("Category", "Subscriptions")] public async Task Can_subscribe_to_a_stream_from_start() diff --git a/tests/SqlStreamStore.AcceptanceTests/IStreamStoreFixture.cs b/tests/SqlStreamStore.AcceptanceTests/IStreamStoreFixture.cs index 037a953c1..1b1a7231e 100644 --- a/tests/SqlStreamStore.AcceptanceTests/IStreamStoreFixture.cs +++ b/tests/SqlStreamStore.AcceptanceTests/IStreamStoreFixture.cs @@ -4,7 +4,7 @@ namespace SqlStreamStore using SqlStreamStore.Infrastructure; using SqlStreamStore.Streams; - public interface IStreamStoreFixture : IDisposable where TReadPage : IReadAllPage + public interface IStreamStoreFixture : IDisposable where TReadPage : IReadAllPage { IStreamStore Store { get; } diff --git a/tests/SqlStreamStore.MsSql.Tests/ExploratoryTests.cs b/tests/SqlStreamStore.MsSql.Tests/ExploratoryTests.cs index 04935b0b3..6c8857278 100644 --- a/tests/SqlStreamStore.MsSql.Tests/ExploratoryTests.cs +++ b/tests/SqlStreamStore.MsSql.Tests/ExploratoryTests.cs @@ -5,7 +5,7 @@ using SqlStreamStore.Streams; using Xunit; - public partial class AcceptanceTests + public partial class AcceptanceTests { [Fact] public async Task Time_to_take_to_read_1000_read_head_positions() diff --git a/tests/SqlStreamStore.MsSql.Tests/StreamStoreAcceptanceTests.ReadLongStreamInPages.cs b/tests/SqlStreamStore.MsSql.Tests/StreamStoreAcceptanceTests.ReadLongStreamInPages.cs index b30f18ee7..31a16d4a9 100644 --- a/tests/SqlStreamStore.MsSql.Tests/StreamStoreAcceptanceTests.ReadLongStreamInPages.cs +++ b/tests/SqlStreamStore.MsSql.Tests/StreamStoreAcceptanceTests.ReadLongStreamInPages.cs @@ -8,7 +8,7 @@ namespace SqlStreamStore using SqlStreamStore.Streams; using Xunit; - public partial class AcceptanceTests + public partial class AcceptanceTests { [Fact] public async Task Given_large_message_stream_can_be_read_back_in_pages() diff --git a/tests/SqlStreamStore.MsSql.V3.Tests/ExploratoryTests.cs b/tests/SqlStreamStore.MsSql.V3.Tests/ExploratoryTests.cs index 5bd9faafe..5dc0eccf2 100644 --- a/tests/SqlStreamStore.MsSql.V3.Tests/ExploratoryTests.cs +++ b/tests/SqlStreamStore.MsSql.V3.Tests/ExploratoryTests.cs @@ -5,7 +5,7 @@ using SqlStreamStore.Streams; using Xunit; - public partial class AcceptanceTests + public partial class AcceptanceTests { [Fact] public async Task Time_to_take_to_read_1000_read_head_positions() diff --git a/tests/SqlStreamStore.MsSql.V3.Tests/MigrationTests.cs b/tests/SqlStreamStore.MsSql.V3.Tests/MigrationTests.cs index 1f5a04d0b..049f42c31 100644 --- a/tests/SqlStreamStore.MsSql.V3.Tests/MigrationTests.cs +++ b/tests/SqlStreamStore.MsSql.V3.Tests/MigrationTests.cs @@ -27,10 +27,10 @@ public async Task Can_migrate() var v2Store = v2Fixture.Store; await v2Store.AppendToStream("stream-1", ExpectedVersion.NoStream, - AcceptanceTests.CreateNewStreamMessages(1, 2, 3)); + AcceptanceTests.CreateNewStreamMessages(1, 2, 3)); await v2Store.AppendToStream("stream-2", ExpectedVersion.NoStream, - AcceptanceTests.CreateNewStreamMessages(1, 2, 3)); + AcceptanceTests.CreateNewStreamMessages(1, 2, 3)); await v2Store.SetStreamMetadata("stream-1", ExpectedVersion.Any, maxAge: 10, maxCount: 20); v2Store.Dispose(); diff --git a/tests/SqlStreamStore.MsSql.V3.Tests/StreamStoreAcceptanceTests.ReadLongStreamInPages.cs b/tests/SqlStreamStore.MsSql.V3.Tests/StreamStoreAcceptanceTests.ReadLongStreamInPages.cs index b30f18ee7..31a16d4a9 100644 --- a/tests/SqlStreamStore.MsSql.V3.Tests/StreamStoreAcceptanceTests.ReadLongStreamInPages.cs +++ b/tests/SqlStreamStore.MsSql.V3.Tests/StreamStoreAcceptanceTests.ReadLongStreamInPages.cs @@ -8,7 +8,7 @@ namespace SqlStreamStore using SqlStreamStore.Streams; using Xunit; - public partial class AcceptanceTests + public partial class AcceptanceTests { [Fact] public async Task Given_large_message_stream_can_be_read_back_in_pages() diff --git a/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreAcceptanceTests.cs b/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreAcceptanceTests.cs index 78982c664..edea5fe04 100644 --- a/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreAcceptanceTests.cs +++ b/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreAcceptanceTests.cs @@ -1,7 +1,6 @@ namespace SqlStreamStore { using System.Threading.Tasks; - using SqlStreamStore.Streams; using Xunit; using Xunit.Abstractions; From 87da7116174886ea8098f3cf23ddae517fe29950 Mon Sep 17 00:00:00 2001 From: Arne Schoonvliet Date: Wed, 20 Apr 2022 17:53:09 +0200 Subject: [PATCH 13/13] WIP --- .../PgSqlScriptsV1/Read.sql | 1 - .../PgSqlScriptsV2/ReadAll.sql | 29 ++--- .../PostgresStreamStore.Append.cs | 2 +- .../PostgresStreamStore.Delete.cs | 2 +- .../PostgresStreamStore.ListStreams.cs | 2 +- .../PostgresStreamStore.Metadata.cs | 2 +- .../PostgresStreamStore.Read.cs | 2 +- .../PostgresStreamStore.ReadAll.cs | 2 +- .../PostgresStreamStore.cs | 110 +++++++++--------- .../Postgres/PostgresContainer.cs | 34 ++---- .../PostgresStreamStoreTests.cs | 15 ++- 11 files changed, 94 insertions(+), 107 deletions(-) diff --git a/src/SqlStreamStore.Postgres/PgSqlScriptsV1/Read.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/Read.sql index f8a1e76a9..3dee4eb07 100644 --- a/src/SqlStreamStore.Postgres/PgSqlScriptsV1/Read.sql +++ b/src/SqlStreamStore.Postgres/PgSqlScriptsV1/Read.sql @@ -42,7 +42,6 @@ BEGIN WHERE (CASE WHEN _forwards THEN __schema__.messages.stream_version >= _version AND id_internal = _stream_id_internal ELSE __schema__.messages.stream_version <= _version AND id_internal = _stream_id_internal END) - AND __schema__.messages.tx_id < txid_snapshot_xmin(txid_current_snapshot()) ORDER BY (CASE WHEN _forwards THEN __schema__.messages.stream_version ELSE -__schema__.messages.stream_version END) diff --git a/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadAll.sql b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadAll.sql index 93e4b63ac..905709e65 100644 --- a/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadAll.sql +++ b/src/SqlStreamStore.Postgres/PgSqlScriptsV2/ReadAll.sql @@ -2,23 +2,20 @@ CREATE OR REPLACE FUNCTION __schema__.read_all( _count INT, _position BIGINT, _forwards BOOLEAN, - _prefetch BOOLEAN + _prefetch BOOLEAN, + _max_position BIGINT ) - RETURNS TABLE( - stream_id VARCHAR(1000), - message_id UUID, - stream_version INT, - "position" BIGINT, - create_utc TIMESTAMP WITH TIME ZONE, - "type" VARCHAR(128), - json_metadata JSONB, - json_data JSONB, - max_age INT - ) + RETURNS SETOF REFCURSOR + AS $F$ + +DECLARE + _messages REFCURSOR := 'messages'; + _txinfo REFCURSOR := 'tx_info'; + BEGIN - RETURN QUERY + OPEN _messages FOR WITH messages AS ( SELECT __schema__.streams.id_original, __schema__.messages.message_id, @@ -34,6 +31,7 @@ BEGIN FROM __schema__.messages INNER JOIN __schema__.streams ON __schema__.messages.stream_id_internal = __schema__.streams.id_internal WHERE (CASE + WHEN _forwards AND _max_position > 0 THEN __schema__.messages.position >= _position AND __schema__.messages.position <= _max_position WHEN _forwards THEN __schema__.messages.position >= _position ELSE __schema__.messages.position <= _position END) ORDER BY @@ -42,6 +40,11 @@ BEGIN LIMIT _count ) SELECT * FROM messages LIMIT _count; + RETURN NEXT _messages; + + OPEN _txinfo FOR + SELECT txid_current_snapshot()::TEXT; + RETURN NEXT _txinfo; END; $F$ LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/src/SqlStreamStore.Postgres/PostgresStreamStore.Append.cs b/src/SqlStreamStore.Postgres/PostgresStreamStore.Append.cs index 493fcc02f..dfd9493de 100644 --- a/src/SqlStreamStore.Postgres/PostgresStreamStore.Append.cs +++ b/src/SqlStreamStore.Postgres/PostgresStreamStore.Append.cs @@ -5,7 +5,7 @@ using System.Threading; using System.Threading.Tasks; using Npgsql; - using SqlStreamStore.PgSqlScriptsV1; + using SqlStreamStore.PgSqlScriptsV2; using SqlStreamStore.Streams; partial class PostgresStreamStore diff --git a/src/SqlStreamStore.Postgres/PostgresStreamStore.Delete.cs b/src/SqlStreamStore.Postgres/PostgresStreamStore.Delete.cs index aeaa1b885..ef139572a 100644 --- a/src/SqlStreamStore.Postgres/PostgresStreamStore.Delete.cs +++ b/src/SqlStreamStore.Postgres/PostgresStreamStore.Delete.cs @@ -4,7 +4,7 @@ using System.Threading; using System.Threading.Tasks; using Npgsql; - using SqlStreamStore.PgSqlScriptsV1; + using SqlStreamStore.PgSqlScriptsV2; using SqlStreamStore.Streams; public partial class PostgresStreamStore diff --git a/src/SqlStreamStore.Postgres/PostgresStreamStore.ListStreams.cs b/src/SqlStreamStore.Postgres/PostgresStreamStore.ListStreams.cs index 3e0a538be..daebd480c 100644 --- a/src/SqlStreamStore.Postgres/PostgresStreamStore.ListStreams.cs +++ b/src/SqlStreamStore.Postgres/PostgresStreamStore.ListStreams.cs @@ -4,7 +4,7 @@ namespace SqlStreamStore using System.Threading; using System.Threading.Tasks; using Npgsql; - using SqlStreamStore.PgSqlScriptsV1; + using SqlStreamStore.PgSqlScriptsV2; using SqlStreamStore.Streams; public partial class PostgresStreamStore diff --git a/src/SqlStreamStore.Postgres/PostgresStreamStore.Metadata.cs b/src/SqlStreamStore.Postgres/PostgresStreamStore.Metadata.cs index 250fda9aa..038c4d55e 100644 --- a/src/SqlStreamStore.Postgres/PostgresStreamStore.Metadata.cs +++ b/src/SqlStreamStore.Postgres/PostgresStreamStore.Metadata.cs @@ -3,7 +3,7 @@ using System.Threading; using System.Threading.Tasks; using Npgsql; - using SqlStreamStore.PgSqlScriptsV1; + using SqlStreamStore.PgSqlScriptsV2; using SqlStreamStore.Streams; partial class PostgresStreamStore diff --git a/src/SqlStreamStore.Postgres/PostgresStreamStore.Read.cs b/src/SqlStreamStore.Postgres/PostgresStreamStore.Read.cs index 6055679d7..bc4d24dfc 100644 --- a/src/SqlStreamStore.Postgres/PostgresStreamStore.Read.cs +++ b/src/SqlStreamStore.Postgres/PostgresStreamStore.Read.cs @@ -9,7 +9,7 @@ using System.Threading; using System.Threading.Tasks; using Npgsql; - using SqlStreamStore.PgSqlScriptsV1; + using SqlStreamStore.PgSqlScriptsV2; using SqlStreamStore.Streams; partial class PostgresStreamStore diff --git a/src/SqlStreamStore.Postgres/PostgresStreamStore.ReadAll.cs b/src/SqlStreamStore.Postgres/PostgresStreamStore.ReadAll.cs index 39304f659..af758d832 100644 --- a/src/SqlStreamStore.Postgres/PostgresStreamStore.ReadAll.cs +++ b/src/SqlStreamStore.Postgres/PostgresStreamStore.ReadAll.cs @@ -11,7 +11,7 @@ using Npgsql; - using SqlStreamStore.PgSqlScriptsV1; + using SqlStreamStore.PgSqlScriptsV2; using SqlStreamStore.Streams; public class PostgresReadAllPage : ReadAllPage diff --git a/src/SqlStreamStore.Postgres/PostgresStreamStore.cs b/src/SqlStreamStore.Postgres/PostgresStreamStore.cs index 2e2a7abdd..bb52e24c3 100644 --- a/src/SqlStreamStore.Postgres/PostgresStreamStore.cs +++ b/src/SqlStreamStore.Postgres/PostgresStreamStore.cs @@ -270,60 +270,60 @@ public string GetMigrationScript() return _schema.Migration; } - protected override async Task HandleGap( - PostgresReadAllPage page, - long fromPositionInclusive, - int maxCount, - bool prefetchJsonData, - CancellationToken cancellationToken) - { - if (page.Messages.Length == 0 || DateTime.UtcNow - page.Messages[page.Messages.Length - 1].CreatedUtc > TimeSpan.FromMinutes(5)) - return page; - - // TODO: FIXIT - // Check for gap between last page and this. - //if (page.Messages[0].Position != fromPositionInclusive) - //{ - // Logger.InfoFormat("Gap detected at lower page boundary. Potentially could have lost {lostMessageCount} events if the gap is transient", page.Messages[0].Position - fromPositionInclusive); - // page = await HandleGap(page, fromPositionInclusive, maxCount, prefetchJsonData, cancellationToken); - // //if (!page.IsEnd || page.Messages.Length == 1) - // // Logger.DebugFormat("Gap detected at lower page boundary. Potentially could have lost {lostMessageCount} events if the gap is transient", page.Messages[0].Position - fromPositionInclusive); - // //page = await ReloadAfterDelay(fromPositionInclusive, maxCount, prefetchJsonData, ReadNext, cancellationToken); - //} - - // check for gap in messages collection - for (int i = 0; i < page.Messages.Length - 1; i++) - { - var expectedNextPosition = page.Messages[i].Position + 1; - if (expectedNextPosition != page.Messages[i + 1].Position) - { - Logger.InfoFormat("Gap detected in " + (page.IsEnd ? "last" : "(NOT the last)") + " page. Returning partial page {fromPosition}-{toPosition}", fromPositionInclusive, fromPositionInclusive + i + 1); - - PostgresReadAllPage requeryPage; - var maxPosition = page.Messages[page.Messages.Length - 1].Position; - do - { - requeryPage = await ReadAllForwardsInternal(fromPositionInclusive, maxCount, prefetchJsonData, cancellationToken, maxPosition); - } while (page.TransactionIds.Intersect(requeryPage.TransactionIds).Any()); - - return requeryPage; - - // switched this to return the partial page, then re-issue load starting at gap - // this speeds up the retry instead of taking a 3 second delay immediately - //var messagesBeforeGap = new StreamMessage[i+1]; - //page.Messages.Take(i+1).ToArray().CopyTo(messagesBeforeGap, 0); - //return new ReadAllPage(page.FromPosition, maxPosition, page.IsEnd, page.Direction, ReadNext, messagesBeforeGap); - } - } - - //ReadAllPage requeryPage; - //var maxPosition = pageWithGap.Messages[pageWithGap.Messages.Length - 1].Position; - //do - //{ - // requeryPage = await ReadAllForwardsInternal(fromPositionInclusive, maxCount, prefetchJsonData, readNext, cancellationToken, maxPosition); - //} while (pageWithGap.TxSnapshot.CurrentTxIds.Intersect(requeryPage.TxSnapshot.CurrentTxIds).Any()); - - return page; - } + //protected override async Task HandleGap( + // PostgresReadAllPage page, + // long fromPositionInclusive, + // int maxCount, + // bool prefetchJsonData, + // CancellationToken cancellationToken) + //{ + // if (page.Messages.Length == 0 || DateTime.UtcNow - page.Messages[page.Messages.Length - 1].CreatedUtc > TimeSpan.FromMinutes(5)) + // return page; + + // // TODO: FIXIT + // // Check for gap between last page and this. + // //if (page.Messages[0].Position != fromPositionInclusive) + // //{ + // // Logger.InfoFormat("Gap detected at lower page boundary. Potentially could have lost {lostMessageCount} events if the gap is transient", page.Messages[0].Position - fromPositionInclusive); + // // page = await HandleGap(page, fromPositionInclusive, maxCount, prefetchJsonData, cancellationToken); + // // //if (!page.IsEnd || page.Messages.Length == 1) + // // // Logger.DebugFormat("Gap detected at lower page boundary. Potentially could have lost {lostMessageCount} events if the gap is transient", page.Messages[0].Position - fromPositionInclusive); + // // //page = await ReloadAfterDelay(fromPositionInclusive, maxCount, prefetchJsonData, ReadNext, cancellationToken); + // //} + + // // check for gap in messages collection + // for (int i = 0; i < page.Messages.Length - 1; i++) + // { + // var expectedNextPosition = page.Messages[i].Position + 1; + // if (expectedNextPosition != page.Messages[i + 1].Position) + // { + // Logger.InfoFormat("Gap detected in " + (page.IsEnd ? "last" : "(NOT the last)") + " page. Returning partial page {fromPosition}-{toPosition}", fromPositionInclusive, fromPositionInclusive + i + 1); + + // PostgresReadAllPage requeryPage; + // var maxPosition = page.Messages[page.Messages.Length - 1].Position; + // do + // { + // requeryPage = await ReadAllForwardsInternal(fromPositionInclusive, maxCount, prefetchJsonData, cancellationToken, maxPosition); + // } while (page.TransactionIds.Intersect(requeryPage.TransactionIds).Any()); + + // return requeryPage; + + // // switched this to return the partial page, then re-issue load starting at gap + // // this speeds up the retry instead of taking a 3 second delay immediately + // //var messagesBeforeGap = new StreamMessage[i+1]; + // //page.Messages.Take(i+1).ToArray().CopyTo(messagesBeforeGap, 0); + // //return new ReadAllPage(page.FromPosition, maxPosition, page.IsEnd, page.Direction, ReadNext, messagesBeforeGap); + // } + // } + + // //ReadAllPage requeryPage; + // //var maxPosition = pageWithGap.Messages[pageWithGap.Messages.Length - 1].Position; + // //do + // //{ + // // requeryPage = await ReadAllForwardsInternal(fromPositionInclusive, maxCount, prefetchJsonData, readNext, cancellationToken, maxPosition); + // //} while (pageWithGap.TxSnapshot.CurrentTxIds.Intersect(requeryPage.TxSnapshot.CurrentTxIds).Any()); + + // return page; + //} } } diff --git a/src/SqlStreamStore.TestUtils/Postgres/PostgresContainer.cs b/src/SqlStreamStore.TestUtils/Postgres/PostgresContainer.cs index 61cb74240..08a4e78a5 100644 --- a/src/SqlStreamStore.TestUtils/Postgres/PostgresContainer.cs +++ b/src/SqlStreamStore.TestUtils/Postgres/PostgresContainer.cs @@ -5,9 +5,10 @@ namespace SqlStreamStore.TestUtils.Postgres using System.Linq; using System.Threading; using System.Threading.Tasks; - //using Ductus.FluentDocker.Builders; + using Ductus.FluentDocker.Extensions; using Ductus.FluentDocker.Model.Containers; using Ductus.FluentDocker.Services; + using Ductus.FluentDocker.Services.Extensions; using Npgsql; using Polly; @@ -17,10 +18,6 @@ public class PostgresContainer : PostgresDatabaseManager private const string Image = "postgres:10.4-alpine"; private const string ContainerName = "sql-stream-store-tests-postgres"; - private readonly int _port; - //public override string ConnectionString => ConnectionStringBuilder.ConnectionString; - - public PostgresContainer(string schema, string databaseName, float cpu = float.MinValue) : base(databaseName) { @@ -28,23 +25,13 @@ public PostgresContainer(string schema, string databaseName, float cpu = float.M var hosts = new Hosts().Discover(); var host = hosts.FirstOrDefault(x => x.IsNative) ?? hosts.FirstOrDefault(x => x.Name == "default"); - - - var b = host.GetRunningContainers().Select(x => x.GetConfiguration(true)) - .Select(x => x.NetworkSettings.Ports).SelectMany(x => x.Values).Where(x => x is not null && x.Any()).SelectMany(x => x).Select(x => x.Port).ToList(); - - var possiblePort = 5432; - while (b.Contains(possiblePort)) - { - possiblePort++; - } - _port = possiblePort; + _containerService = host.Create(Image, false, new ContainerCreateParams { Cpus = cpu, Name = $"{schema}-{ContainerName}", - PortMappings = new[] { $"{_port}:5432" } + PortMappings = new[] { "5432" } }, false, true, command: "-N 500"); @@ -55,18 +42,16 @@ public PostgresContainer(string schema, string databaseName, float cpu = float.M // .UseImage(Image) // .KeepRunning() // .ReuseIfExists() - // //.ExposePort(Port, Port) + // .ExposePort(Port, Port) // .Command("-N", "500") // .WithParentCGroup() // .Build(); - - //var c = _containerService.GetConfiguration(true); - //var d = c1.GetConfiguration(true); } public async Task Start(CancellationToken cancellationToken = default) { _containerService.Start(); + _containerService.WaitForRunning(); await Policy .Handle() @@ -82,19 +67,20 @@ await Policy public override string GenerateConnectionString(string applicationName = "default") { + return new NpgsqlConnectionStringBuilder(ConnectionStringBuilder.ConnectionString) { - ApplicationName = applicationName + ApplicationName = applicationName, + Port = _containerService.ToHostExposedEndpoint("5432/tcp").Port }.ConnectionString; } - private NpgsqlConnectionStringBuilder ConnectionStringBuilder => new NpgsqlConnectionStringBuilder + private NpgsqlConnectionStringBuilder ConnectionStringBuilder => new() { Database = DatabaseName, Password = Environment.OSVersion.IsWindows() ? "password" : null, - Port = _port, Username = "postgres", Host = "localhost", Pooling = true, diff --git a/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreTests.cs b/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreTests.cs index a048e7137..9f940a278 100644 --- a/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreTests.cs +++ b/tests/SqlStreamStore.Postgres.Tests/PostgresStreamStoreTests.cs @@ -41,8 +41,6 @@ public async Task Can_Handle_High_Load_Gaps() await writerStore.CreateSchemaIfNotExists(); - var cancellationTokenSource = new CancellationTokenSource(); - var receiveMessages = new TaskCompletionSource(); List receivedMessages = new List(); @@ -52,7 +50,7 @@ public async Task Can_Handle_High_Load_Gaps() { _testOutputHelper.WriteLine($"Received message {message.StreamId} " + $"{message.StreamVersion} {message.Position}"); - if (message.Position >= 100) + if (message.Position >= 200) receiveMessages.SetResult(message); else receivedMessages.Add(message); @@ -60,9 +58,10 @@ public async Task Can_Handle_High_Load_Gaps() return Task.CompletedTask; }); - var appendTask = AppendMessages(writerStore, 200, cancellationTokenSource.Token); - await Task.WhenAny(receiveMessages.Task.WithTimeout(100000), appendTask); - cancellationTokenSource.Cancel(); + all.MaxCountPerRead = 500; + + await AppendMessages(writerStore, 1000); + await receiveMessages.Task.WithTimeout(1000000); receivedMessages.Count.ShouldBe(100); } @@ -78,12 +77,12 @@ private static PostgresStreamStoreSettings GetSettings(string connectionString) }; } - private static async Task AppendMessages(IStreamStore streamStore, int numberOfEvents, CancellationToken ct) + private static async Task AppendMessages(IStreamStore streamStore, int numberOfEvents) { await Task.WhenAll(Enumerable.Range(0, numberOfEvents).Select(_ => { var newMessage = new NewStreamMessage(Guid.NewGuid(), "MyEvent", "{}"); - return streamStore.AppendToStream(Guid.NewGuid().ToString(), ExpectedVersion.Any, newMessage, ct); + return streamStore.AppendToStream(Guid.NewGuid().ToString(), ExpectedVersion.Any, newMessage); })); }