Implement parsing

Hcs.WebApp/BackgroundServices/DataParsers/DataParserBase.cs

@@ -1,12 +1,17 @@
-using Hcs.Broker;
+using Hcs.WebApp.Data.Hcs;
-using Hcs.WebApp.Data.Hcs;
+using Hcs.WebApp.Services;
 
 namespace Hcs.WebApp.BackgroundServices.DataParsers
 {
-    public abstract class DataParserBase(IServiceScope scope, Operation operation) : IDataParser
+    public abstract class DataParserBase(IServiceScope scope, Operation operation, IWebHostEnvironment webHostEnvironment) : IDataParser
     {
         protected readonly IServiceScope scope = scope;
         protected readonly Operation operation = operation;
+        protected readonly IWebHostEnvironment webHostEnvironment = webHostEnvironment;
+
+        private FileToParseService? fileToParseService;
+
+        protected FileToParseService FileToParseService => fileToParseService ??= scope.ServiceProvider.GetRequiredService<FileToParseService>();
 
         public abstract Task ParseAsync();
     }

Hcs.WebApp/BackgroundServices/DataParsers/DataParserFactory.cs

@@ -1,15 +1,14 @@
-using Hcs.Broker;
+using Hcs.WebApp.Data.Hcs;
-using Hcs.WebApp.Data.Hcs;
 
 namespace Hcs.WebApp.BackgroundServices.DataParsers
 {
     public class DataParserFactory
     {
-        public IDataParser CreateDataParser(IServiceScope scope, Operation operation)
+        public IDataParser CreateDataParser(IServiceScope scope, Operation operation, IWebHostEnvironment webHostEnvironment)
         {
             return operation.Type switch
             {
-                Operation.OperationType.ParseHousesData_15_7_0_1 => new HousesDataParser_15_7_0_1(scope, operation),
+                Operation.OperationType.ParseHousesData_15_7_0_1 => new HousesDataParser_15_7_0_1(scope, operation, webHostEnvironment),
 
                 Operation.OperationType.NsiCommon_ExportNsiItem_15_7_0_1 or
                 Operation.OperationType.Nsi_ExportNsiItem_15_7_0_1 => throw new ArgumentException($"Нельзя использовать операцию с типом {operation.Type} для парсинга")

Hcs.WebApp/BackgroundServices/DataParsers/HousesDataParser_15_7_0_1.cs

@@ -1,12 +1,72 @@
 using Hcs.WebApp.Data.Hcs;
 using Hcs.WebApp.Services;
 using Microsoft.EntityFrameworkCore;
+using Sylvan.Data.Excel;
 
 namespace Hcs.WebApp.BackgroundServices.DataParsers
 {
-    public class HousesDataParser_15_7_0_1(IServiceScope scope, Operation operation) : DataParserBase(scope, operation)
+    public class HousesDataParser_15_7_0_1(
+        IServiceScope scope,
+        Operation operation,
+        IWebHostEnvironment webHostEnvironment) : DataParserBase(scope, operation, webHostEnvironment)
     {
         public override async Task ParseAsync()
+        {
+            await ParseFile();
+            await CompleteOperation();
+        }
+
+        private async Task ParseFile()
+        {
+            const int batchMaxSize = 100;
+            const string mkd = "Многоквартирный";
+
+            var batch = new List<House>();
+            var houseService = scope.ServiceProvider.GetRequiredService<HouseService>();
+
+            var fileToParse = await FileToParseService.GetFileToParseByOperationIdAsync(operation.Id);
+            var fullPath = Path.Combine(webHostEnvironment.WebRootPath, fileToParse.Path);
+            using var stream = new FileStream(fullPath, FileMode.Open);
+            using ExcelDataReader reader = ExcelDataReader.Create(stream, ExcelWorkbookType.ExcelXml);
+            while (reader.Read())
+            {
+                var fiasId = reader.GetString(2);
+                var houseType = reader.GetString(7);
+                var roomNum = reader.GetString(13);
+                var hcsId = reader.GetString(25);
+
+                if (!string.IsNullOrEmpty(hcsId))
+                {
+                    if (string.IsNullOrEmpty(roomNum))
+                    {
+                        var isMkd = houseType == mkd;
+                        batch.Add(new House()
+                        {
+                            FiasId = Guid.Parse(fiasId),
+                            HcsId = Guid.Parse(hcsId),
+                            IsMkd = isMkd,
+                            IsZhd = !isMkd,
+                            SyncedAt = DateTime.UtcNow,
+                            LastSyncOperationId = operation.Id
+                        });
+                    }
+                }
+
+                if (batch.Count >= batchMaxSize)
+                {
+                    await houseService.UpsertHouses(batch);
+
+                    batch.Clear();
+                }
+            }
+
+            if (batch.Count > 0)
+            {
+                await houseService.UpsertHouses(batch);
+            }
+        }
+
+        private async Task CompleteOperation()
         {
             var headquartersService = scope.ServiceProvider.GetRequiredService<HeadquartersService>();
             var fileToParseService = scope.ServiceProvider.GetRequiredService<FileToParseService>();

Hcs.WebApp/BackgroundServices/DataParsingService.cs

@@ -7,13 +7,15 @@ namespace Hcs.WebApp.BackgroundServices
     public class DataParsingService(
         DataParsingState state,
         DataParserFactory dataParserFactory,
-        IServiceScopeFactory scopeFactory) : BackgroundService
+        IServiceScopeFactory scopeFactory,
+        IWebHostEnvironment webHostEnvironment) : BackgroundService
     {
         private const int SLEEP_TIME = 30000;
 
         private readonly DataParsingState state = state;
         private readonly DataParserFactory dataParserFactory = dataParserFactory;
         private readonly IServiceScopeFactory scopeFactory = scopeFactory;
+        private readonly IWebHostEnvironment webHostEnvironment = webHostEnvironment;
 
         protected override async Task ExecuteAsync(CancellationToken stoppingToken)
         {
@@ -36,7 +38,7 @@ namespace Hcs.WebApp.BackgroundServices
 
                         state.InvokeOnOperationStarted(operation.Id, operation.CampaignId, startedAt);
 
-                        var dataParser = dataParserFactory.CreateDataParser(scope, operation);
+                        var dataParser = dataParserFactory.CreateDataParser(scope, operation, webHostEnvironment);
                         await dataParser.ParseAsync();
                     }
                     catch (Exception e)

Hcs.WebApp/Components/Dialogs/StartParsing.razor

@@ -77,7 +77,7 @@
                 root.GetProperty("path").GetString(),
                 root.GetProperty("fileName").GetString(),
                 UploaderId,
-                DateTime.Now);
+                DateTime.UtcNow);
             fileToParseId = fileToParse.Id;
         }
         catch (Exception e)

Hcs.WebApp/Hcs.WebApp.csproj

@@ -16,6 +16,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="EFCore.BulkExtensions" Version="8.1.3" />
     <PackageReference Include="Microsoft.AspNetCore.Diagnostics.EntityFrameworkCore" Version="8.0.20" />
     <PackageReference Include="Microsoft.AspNetCore.HeaderPropagation" Version="8.0.20" />
     <PackageReference Include="Microsoft.AspNetCore.Identity.EntityFrameworkCore" Version="8.0.20" />
@@ -28,7 +29,6 @@
     </PackageReference>
     <PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="8.0.7" />
     <PackageReference Include="Radzen.Blazor" Version="8.0.4" />
-    <PackageReference Include="Sylvan.Data" Version="0.2.16" />
     <PackageReference Include="Sylvan.Data.Excel" Version="0.5.2" />
   </ItemGroup>
 

Hcs.WebApp/Services/FileToParseService.cs

@@ -27,6 +27,12 @@ namespace Hcs.WebApp.Services
             return fileToParse;
         }
 
+        public async Task<FileToParse> GetFileToParseByOperationIdAsync(int operationId)
+        {
+            using var context = GetNewContext();
+            return await GetFileToParseByOperationIdAsync(context, operationId);
+        }
+
         public async Task<FileToParse> GetFileToParseByOperationIdAsync(HcsDbContext context, int operationId)
         {
             return await context.FilesToParse.SingleAsync(x => x.LastParseOperationId == operationId);

Hcs.WebApp/Services/HouseService.cs

@@ -1,4 +1,5 @@
-using Hcs.WebApp.Data.Hcs;
+using EFCore.BulkExtensions;
+using Hcs.WebApp.Data.Hcs;
 using Microsoft.EntityFrameworkCore;
 
 namespace Hcs.WebApp.Services
@@ -10,5 +11,21 @@ namespace Hcs.WebApp.Services
             using var context = GetNewContext();
             return await context.Houses.ToListAsync();
         }
+
+        public async Task UpsertHouses(IEnumerable<House> houses)
+        {
+            using var context = GetNewContext();
+            await context.BulkInsertOrUpdateAsync(houses, new BulkConfig()
+            {
+                PropertiesToExcludeOnUpdate =
+                [
+                    nameof(House.ThirdPartyId)
+                ],
+                UpdateByProperties =
+                [
+                    nameof(House.HcsId)
+                ]
+            });
+        }
     }
 }