diff --git a/Hcs.WebApp/BackgroundServices/DataParsers/DataParserBase.cs b/Hcs.WebApp/BackgroundServices/DataParsers/DataParserBase.cs index 21533f8..cf5935c 100644 --- a/Hcs.WebApp/BackgroundServices/DataParsers/DataParserBase.cs +++ b/Hcs.WebApp/BackgroundServices/DataParsers/DataParserBase.cs @@ -1,12 +1,17 @@ -using Hcs.Broker; -using Hcs.WebApp.Data.Hcs; +using Hcs.WebApp.Data.Hcs; +using Hcs.WebApp.Services; namespace Hcs.WebApp.BackgroundServices.DataParsers { - public abstract class DataParserBase(IServiceScope scope, Operation operation) : IDataParser + public abstract class DataParserBase(IServiceScope scope, Operation operation, IWebHostEnvironment webHostEnvironment) : IDataParser { protected readonly IServiceScope scope = scope; protected readonly Operation operation = operation; + protected readonly IWebHostEnvironment webHostEnvironment = webHostEnvironment; + + private FileToParseService? fileToParseService; + + protected FileToParseService FileToParseService => fileToParseService ??= scope.ServiceProvider.GetRequiredService(); public abstract Task ParseAsync(); } diff --git a/Hcs.WebApp/BackgroundServices/DataParsers/DataParserFactory.cs b/Hcs.WebApp/BackgroundServices/DataParsers/DataParserFactory.cs index a831053..b88676b 100644 --- a/Hcs.WebApp/BackgroundServices/DataParsers/DataParserFactory.cs +++ b/Hcs.WebApp/BackgroundServices/DataParsers/DataParserFactory.cs @@ -1,15 +1,14 @@ -using Hcs.Broker; -using Hcs.WebApp.Data.Hcs; +using Hcs.WebApp.Data.Hcs; namespace Hcs.WebApp.BackgroundServices.DataParsers { public class DataParserFactory { - public IDataParser CreateDataParser(IServiceScope scope, Operation operation) + public IDataParser CreateDataParser(IServiceScope scope, Operation operation, IWebHostEnvironment webHostEnvironment) { return operation.Type switch { - Operation.OperationType.ParseHousesData_15_7_0_1 => new HousesDataParser_15_7_0_1(scope, operation), + Operation.OperationType.ParseHousesData_15_7_0_1 => new HousesDataParser_15_7_0_1(scope, operation, webHostEnvironment), Operation.OperationType.NsiCommon_ExportNsiItem_15_7_0_1 or Operation.OperationType.Nsi_ExportNsiItem_15_7_0_1 => throw new ArgumentException($"Нельзя использовать операцию с типом {operation.Type} для парсинга") diff --git a/Hcs.WebApp/BackgroundServices/DataParsers/HousesDataParser_15_7_0_1.cs b/Hcs.WebApp/BackgroundServices/DataParsers/HousesDataParser_15_7_0_1.cs index 39303d7..9862398 100644 --- a/Hcs.WebApp/BackgroundServices/DataParsers/HousesDataParser_15_7_0_1.cs +++ b/Hcs.WebApp/BackgroundServices/DataParsers/HousesDataParser_15_7_0_1.cs @@ -1,12 +1,72 @@ using Hcs.WebApp.Data.Hcs; using Hcs.WebApp.Services; using Microsoft.EntityFrameworkCore; +using Sylvan.Data.Excel; namespace Hcs.WebApp.BackgroundServices.DataParsers { - public class HousesDataParser_15_7_0_1(IServiceScope scope, Operation operation) : DataParserBase(scope, operation) + public class HousesDataParser_15_7_0_1( + IServiceScope scope, + Operation operation, + IWebHostEnvironment webHostEnvironment) : DataParserBase(scope, operation, webHostEnvironment) { public override async Task ParseAsync() + { + await ParseFile(); + await CompleteOperation(); + } + + private async Task ParseFile() + { + const int batchMaxSize = 100; + const string mkd = "Многоквартирный"; + + var batch = new List(); + var houseService = scope.ServiceProvider.GetRequiredService(); + + var fileToParse = await FileToParseService.GetFileToParseByOperationIdAsync(operation.Id); + var fullPath = Path.Combine(webHostEnvironment.WebRootPath, fileToParse.Path); + using var stream = new FileStream(fullPath, FileMode.Open); + using ExcelDataReader reader = ExcelDataReader.Create(stream, ExcelWorkbookType.ExcelXml); + while (reader.Read()) + { + var fiasId = reader.GetString(2); + var houseType = reader.GetString(7); + var roomNum = reader.GetString(13); + var hcsId = reader.GetString(25); + + if (!string.IsNullOrEmpty(hcsId)) + { + if (string.IsNullOrEmpty(roomNum)) + { + var isMkd = houseType == mkd; + batch.Add(new House() + { + FiasId = Guid.Parse(fiasId), + HcsId = Guid.Parse(hcsId), + IsMkd = isMkd, + IsZhd = !isMkd, + SyncedAt = DateTime.UtcNow, + LastSyncOperationId = operation.Id + }); + } + } + + if (batch.Count >= batchMaxSize) + { + await houseService.UpsertHouses(batch); + + batch.Clear(); + } + } + + if (batch.Count > 0) + { + await houseService.UpsertHouses(batch); + } + } + + private async Task CompleteOperation() { var headquartersService = scope.ServiceProvider.GetRequiredService(); var fileToParseService = scope.ServiceProvider.GetRequiredService(); diff --git a/Hcs.WebApp/BackgroundServices/DataParsingService.cs b/Hcs.WebApp/BackgroundServices/DataParsingService.cs index 9449a44..3b88268 100644 --- a/Hcs.WebApp/BackgroundServices/DataParsingService.cs +++ b/Hcs.WebApp/BackgroundServices/DataParsingService.cs @@ -7,13 +7,15 @@ namespace Hcs.WebApp.BackgroundServices public class DataParsingService( DataParsingState state, DataParserFactory dataParserFactory, - IServiceScopeFactory scopeFactory) : BackgroundService + IServiceScopeFactory scopeFactory, + IWebHostEnvironment webHostEnvironment) : BackgroundService { private const int SLEEP_TIME = 30000; private readonly DataParsingState state = state; private readonly DataParserFactory dataParserFactory = dataParserFactory; private readonly IServiceScopeFactory scopeFactory = scopeFactory; + private readonly IWebHostEnvironment webHostEnvironment = webHostEnvironment; protected override async Task ExecuteAsync(CancellationToken stoppingToken) { @@ -36,7 +38,7 @@ namespace Hcs.WebApp.BackgroundServices state.InvokeOnOperationStarted(operation.Id, operation.CampaignId, startedAt); - var dataParser = dataParserFactory.CreateDataParser(scope, operation); + var dataParser = dataParserFactory.CreateDataParser(scope, operation, webHostEnvironment); await dataParser.ParseAsync(); } catch (Exception e) diff --git a/Hcs.WebApp/Components/Dialogs/StartParsing.razor b/Hcs.WebApp/Components/Dialogs/StartParsing.razor index f235a9f..d0214de 100644 --- a/Hcs.WebApp/Components/Dialogs/StartParsing.razor +++ b/Hcs.WebApp/Components/Dialogs/StartParsing.razor @@ -77,7 +77,7 @@ root.GetProperty("path").GetString(), root.GetProperty("fileName").GetString(), UploaderId, - DateTime.Now); + DateTime.UtcNow); fileToParseId = fileToParse.Id; } catch (Exception e) diff --git a/Hcs.WebApp/Hcs.WebApp.csproj b/Hcs.WebApp/Hcs.WebApp.csproj index da14e1a..d5da7ea 100644 --- a/Hcs.WebApp/Hcs.WebApp.csproj +++ b/Hcs.WebApp/Hcs.WebApp.csproj @@ -16,6 +16,7 @@ + @@ -28,7 +29,6 @@ - diff --git a/Hcs.WebApp/Services/FileToParseService.cs b/Hcs.WebApp/Services/FileToParseService.cs index f8bb340..3e128dc 100644 --- a/Hcs.WebApp/Services/FileToParseService.cs +++ b/Hcs.WebApp/Services/FileToParseService.cs @@ -27,6 +27,12 @@ namespace Hcs.WebApp.Services return fileToParse; } + public async Task GetFileToParseByOperationIdAsync(int operationId) + { + using var context = GetNewContext(); + return await GetFileToParseByOperationIdAsync(context, operationId); + } + public async Task GetFileToParseByOperationIdAsync(HcsDbContext context, int operationId) { return await context.FilesToParse.SingleAsync(x => x.LastParseOperationId == operationId); diff --git a/Hcs.WebApp/Services/HouseService.cs b/Hcs.WebApp/Services/HouseService.cs index 03af57f..12329ae 100644 --- a/Hcs.WebApp/Services/HouseService.cs +++ b/Hcs.WebApp/Services/HouseService.cs @@ -1,4 +1,5 @@ -using Hcs.WebApp.Data.Hcs; +using EFCore.BulkExtensions; +using Hcs.WebApp.Data.Hcs; using Microsoft.EntityFrameworkCore; namespace Hcs.WebApp.Services @@ -10,5 +11,21 @@ namespace Hcs.WebApp.Services using var context = GetNewContext(); return await context.Houses.ToListAsync(); } + + public async Task UpsertHouses(IEnumerable houses) + { + using var context = GetNewContext(); + await context.BulkInsertOrUpdateAsync(houses, new BulkConfig() + { + PropertiesToExcludeOnUpdate = + [ + nameof(House.ThirdPartyId) + ], + UpdateByProperties = + [ + nameof(House.HcsId) + ] + }); + } } }