From 4785d162a23653f7ed122248c4e2a5a39fc1e75e Mon Sep 17 00:00:00 2001 From: Octol1ttle Date: Mon, 3 Feb 2025 16:58:57 +0500 Subject: [PATCH] Handle temporary files being present when loading guild data (#345) This PR fixes catastrophic guild data loading errors that appear when there are lingering temporary files. In normal operation, temporary files are deleted as soon as they are copied to the main file. It is also expected that temporary files are valid JSON files. However, due to a yesterday's DoS attack, something:tm: happened and a bunch of empty temporary files got written to disk. When Octobot recovered from the attack, it was unable to load any guild data because of the temporary files. This PR addresses this issue by changing the data loading logic: 1) Check if there's a temporary file. If it exists, try loading it. 2) If it is successfully loaded, move the temp file to the main file and resume operation as normal 3) If it could not be loaded, try loading the main file 4) If it is successfully loaded, delete the temporary file and resume operation as normal 5) If it is not, throw an error (like before) This PR was tested on production data and managed to load every guild without errors. Signed-off-by: Octol1ttle --- .../GuildScheduledEventExtensions.cs | 2 +- .../Services/GuildDataService.cs | 189 +++++++++++++----- 2 files changed, 144 insertions(+), 47 deletions(-) diff --git a/TeamOctolings.Octobot/Extensions/GuildScheduledEventExtensions.cs b/TeamOctolings.Octobot/Extensions/GuildScheduledEventExtensions.cs index b8eb2d1..7822d9b 100644 --- a/TeamOctolings.Octobot/Extensions/GuildScheduledEventExtensions.cs +++ b/TeamOctolings.Octobot/Extensions/GuildScheduledEventExtensions.cs @@ -10,7 +10,7 @@ public static class GuildScheduledEventExtensions out string? location) { endTime = default; - location = default; + location = null; if (!scheduledEvent.EntityMetadata.AsOptional().IsDefined(out var metadata)) { return new ArgumentNullError(nameof(scheduledEvent.EntityMetadata)); diff --git a/TeamOctolings.Octobot/Services/GuildDataService.cs b/TeamOctolings.Octobot/Services/GuildDataService.cs index a7af7c9..88edb5f 100644 --- a/TeamOctolings.Octobot/Services/GuildDataService.cs +++ b/TeamOctolings.Octobot/Services/GuildDataService.cs @@ -75,78 +75,48 @@ public sealed class GuildDataService : BackgroundService { var path = $"GuildData/{guildId}"; var memberDataPath = $"{path}/MemberData"; + var settingsPath = $"{path}/Settings.json"; + var scheduledEventsPath = $"{path}/ScheduledEvents.json"; MigrateDataDirectory(guildId, path); Directory.CreateDirectory(path); - if (!File.Exists(settingsPath)) - { - await File.WriteAllTextAsync(settingsPath, "{}", ct); - } - - if (!File.Exists(scheduledEventsPath)) - { - await File.WriteAllTextAsync(scheduledEventsPath, "{}", ct); - } - var dataLoadFailed = false; - await using var settingsStream = File.OpenRead(settingsPath); - JsonNode? jsonSettings = null; - try - { - jsonSettings = await JsonNode.ParseAsync(settingsStream, cancellationToken: ct); - } - catch (Exception e) - { - _logger.LogError(e, "Guild settings load failed: {Path}", settingsPath); - dataLoadFailed = true; - } - + var jsonSettings = await LoadGuildSettings(settingsPath, ct); if (jsonSettings is not null) { FixJsonSettings(jsonSettings); } - - await using var eventsStream = File.OpenRead(scheduledEventsPath); - Dictionary? events = null; - try + else { - events = await JsonSerializer.DeserializeAsync>( - eventsStream, cancellationToken: ct); + dataLoadFailed = true; } - catch (Exception e) + + var events = await LoadScheduledEvents(scheduledEventsPath, ct); + if (events is null) { - _logger.LogError(e, "Guild scheduled events load failed: {Path}", scheduledEventsPath); dataLoadFailed = true; } var memberData = new Dictionary(); - foreach (var dataFileInfo in Directory.CreateDirectory(memberDataPath).GetFiles()) + foreach (var dataFileInfo in Directory.CreateDirectory(memberDataPath).GetFiles() + .Where(dataFileInfo => + !memberData.ContainsKey( + ulong.Parse(dataFileInfo.Name.Replace(".json", "").Replace(".tmp", ""))))) { - await using var dataStream = dataFileInfo.OpenRead(); - MemberData? data; - try + var data = await LoadMemberData(dataFileInfo, memberDataPath, true, ct); + + if (data == null) { - data = await JsonSerializer.DeserializeAsync(dataStream, cancellationToken: ct); - } - catch (Exception e) - { - _logger.LogError(e, "Member data load failed: {MemberDataPath}/{FileName}", memberDataPath, - dataFileInfo.Name); dataLoadFailed = true; continue; } - if (data is null) - { - continue; - } - - memberData.Add(data.Id, data); + memberData.TryAdd(data.Id, data); } var finalData = new GuildData( @@ -160,6 +130,133 @@ public sealed class GuildDataService : BackgroundService return finalData; } + private async Task LoadMemberData(FileInfo dataFileInfo, string memberDataPath, bool loadTmp, + CancellationToken ct = default) + { + MemberData? data; + var temporaryPath = $"{dataFileInfo.FullName}.tmp"; + var usedInfo = loadTmp && File.Exists(temporaryPath) ? new FileInfo(temporaryPath) : dataFileInfo; + + var isTmp = usedInfo.Extension is ".tmp"; + try + { + await using var dataStream = usedInfo.OpenRead(); + data = await JsonSerializer.DeserializeAsync(dataStream, cancellationToken: ct); + if (isTmp) + { + usedInfo.CopyTo(usedInfo.FullName.Replace(".tmp", ""), true); + usedInfo.Delete(); + } + } + catch (Exception e) + { + if (isTmp) + { + _logger.LogWarning(e, + "Unable to load temporary member data file, deleting: {MemberDataPath}/{FileName}", memberDataPath, + usedInfo.Name); + usedInfo.Delete(); + return await LoadMemberData(dataFileInfo, memberDataPath, false, ct); + } + + _logger.LogError(e, "Member data load failed: {MemberDataPath}/{FileName}", memberDataPath, + usedInfo.Name); + return null; + } + + return data; + } + + private async Task?> LoadScheduledEvents(string scheduledEventsPath, + CancellationToken ct = default) + { + var tempScheduledEventsPath = $"{scheduledEventsPath}.tmp"; + + if (!File.Exists(scheduledEventsPath) && !File.Exists(tempScheduledEventsPath)) + { + return new Dictionary(); + } + + if (File.Exists(tempScheduledEventsPath)) + { + _logger.LogWarning("Found temporary scheduled events file, will try to parse and copy to main: ${Path}", + tempScheduledEventsPath); + try + { + await using var tempEventsStream = File.OpenRead(tempScheduledEventsPath); + var events = await JsonSerializer.DeserializeAsync>( + tempEventsStream, cancellationToken: ct); + File.Copy(tempScheduledEventsPath, scheduledEventsPath, true); + File.Delete(tempScheduledEventsPath); + + _logger.LogInformation("Successfully loaded temporary scheduled events file: ${Path}", + tempScheduledEventsPath); + return events; + } + catch (Exception e) + { + _logger.LogError(e, "Unable to load temporary scheduled events file: {Path}, deleting", + tempScheduledEventsPath); + File.Delete(tempScheduledEventsPath); + } + } + + try + { + await using var eventsStream = File.OpenRead(scheduledEventsPath); + return await JsonSerializer.DeserializeAsync>( + eventsStream, cancellationToken: ct); + } + catch (Exception e) + { + _logger.LogError(e, "Guild scheduled events load failed: {Path}", scheduledEventsPath); + return null; + } + } + + private async Task LoadGuildSettings(string settingsPath, CancellationToken ct = default) + { + var tempSettingsPath = $"{settingsPath}.tmp"; + + if (!File.Exists(settingsPath) && !File.Exists(tempSettingsPath)) + { + return new JsonObject(); + } + + if (File.Exists(tempSettingsPath)) + { + _logger.LogWarning("Found temporary settings file, will try to parse and copy to main: ${Path}", + tempSettingsPath); + try + { + await using var tempSettingsStream = File.OpenRead(tempSettingsPath); + var jsonSettings = await JsonNode.ParseAsync(tempSettingsStream, cancellationToken: ct); + + File.Copy(tempSettingsPath, settingsPath, true); + File.Delete(tempSettingsPath); + + _logger.LogInformation("Successfully loaded temporary settings file: ${Path}", tempSettingsPath); + return jsonSettings; + } + catch (Exception e) + { + _logger.LogError(e, "Unable to load temporary settings file: {Path}, deleting", tempSettingsPath); + File.Delete(tempSettingsPath); + } + } + + try + { + await using var settingsStream = File.OpenRead(settingsPath); + return await JsonNode.ParseAsync(settingsStream, cancellationToken: ct); + } + catch (Exception e) + { + _logger.LogError(e, "Guild settings load failed: {Path}", settingsPath); + return null; + } + } + private void MigrateDataDirectory(Snowflake guildId, string newPath) { var oldPath = $"{guildId}";