kshkulev/hcs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement parsing

Browse Source
This commit is contained in:
HOME-LAPTOP\kshkulev
2025-11-22 11:10:03 +09:00
parent 8095bf8ebc
commit dfb60cb9f0
8 changed files with 102 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
Hcs.WebApp/BackgroundServices/DataParsers/DataParserBase.cs
View File

@ -1,12 +1,17 @@
using Hcs.Broker;
using Hcs.WebApp.Data.Hcs;
using Hcs.WebApp.Data.Hcs;
using Hcs.WebApp.Services;
namespace Hcs.WebApp.BackgroundServices.DataParsers
{
public abstract class DataParserBase(IServiceScope scope, Operation operation) : IDataParser
public abstract class DataParserBase(IServiceScope scope, Operation operation, IWebHostEnvironment webHostEnvironment) : IDataParser
{
protected readonly IServiceScope scope = scope;
protected readonly Operation operation = operation;
protected readonly IWebHostEnvironment webHostEnvironment = webHostEnvironment;
private FileToParseService? fileToParseService;
protected FileToParseService FileToParseService => fileToParseService ??= scope.ServiceProvider.GetRequiredService<FileToParseService>();
public abstract Task ParseAsync();
}

7
Hcs.WebApp/BackgroundServices/DataParsers/DataParserFactory.cs
View File

@ -1,15 +1,14 @@
using Hcs.Broker;
using Hcs.WebApp.Data.Hcs;
using Hcs.WebApp.Data.Hcs;
namespace Hcs.WebApp.BackgroundServices.DataParsers
{
public class DataParserFactory
{
public IDataParser CreateDataParser(IServiceScope scope, Operation operation)
public IDataParser CreateDataParser(IServiceScope scope, Operation operation, IWebHostEnvironment webHostEnvironment)
{
return operation.Type switch
{
Operation.OperationType.ParseHousesData_15_7_0_1 => new HousesDataParser_15_7_0_1(scope, operation),
Operation.OperationType.ParseHousesData_15_7_0_1 => new HousesDataParser_15_7_0_1(scope, operation, webHostEnvironment),
Operation.OperationType.NsiCommon_ExportNsiItem_15_7_0_1 or
Operation.OperationType.Nsi_ExportNsiItem_15_7_0_1 => throw new ArgumentException($"Нельзя использовать операцию с типом {operation.Type} для парсинга")

62
Hcs.WebApp/BackgroundServices/DataParsers/HousesDataParser_15_7_0_1.cs
View File

@ -1,12 +1,72 @@
using Hcs.WebApp.Data.Hcs;
using Hcs.WebApp.Services;
using Microsoft.EntityFrameworkCore;
using Sylvan.Data.Excel;
namespace Hcs.WebApp.BackgroundServices.DataParsers
{
public class HousesDataParser_15_7_0_1(IServiceScope scope, Operation operation) : DataParserBase(scope, operation)
public class HousesDataParser_15_7_0_1(
IServiceScope scope,
Operation operation,
IWebHostEnvironment webHostEnvironment) : DataParserBase(scope, operation, webHostEnvironment)
{
public override async Task ParseAsync()
{
await ParseFile();
await CompleteOperation();
}
private async Task ParseFile()
{
const int batchMaxSize = 100;
const string mkd = "Многоквартирный";
var batch = new List<House>();
var houseService = scope.ServiceProvider.GetRequiredService<HouseService>();
var fileToParse = await FileToParseService.GetFileToParseByOperationIdAsync(operation.Id);
var fullPath = Path.Combine(webHostEnvironment.WebRootPath, fileToParse.Path);
using var stream = new FileStream(fullPath, FileMode.Open);
using ExcelDataReader reader = ExcelDataReader.Create(stream, ExcelWorkbookType.ExcelXml);
while (reader.Read())
{
var fiasId = reader.GetString(2);
var houseType = reader.GetString(7);
var roomNum = reader.GetString(13);
var hcsId = reader.GetString(25);
if (!string.IsNullOrEmpty(hcsId))
{
if (string.IsNullOrEmpty(roomNum))
{
var isMkd = houseType == mkd;
batch.Add(new House()
{
FiasId = Guid.Parse(fiasId),
HcsId = Guid.Parse(hcsId),
IsMkd = isMkd,
IsZhd = !isMkd,
SyncedAt = DateTime.UtcNow,
LastSyncOperationId = operation.Id
});
}
}
if (batch.Count >= batchMaxSize)
{
await houseService.UpsertHouses(batch);
batch.Clear();
}
}
if (batch.Count > 0)
{
await houseService.UpsertHouses(batch);
}
}
private async Task CompleteOperation()
{
var headquartersService = scope.ServiceProvider.GetRequiredService<HeadquartersService>();
var fileToParseService = scope.ServiceProvider.GetRequiredService<FileToParseService>();

6
Hcs.WebApp/BackgroundServices/DataParsingService.cs
View File

@ -7,13 +7,15 @@ namespace Hcs.WebApp.BackgroundServices
public class DataParsingService(
DataParsingState state,
DataParserFactory dataParserFactory,
IServiceScopeFactory scopeFactory) : BackgroundService
IServiceScopeFactory scopeFactory,
IWebHostEnvironment webHostEnvironment) : BackgroundService
{
private const int SLEEP_TIME = 30000;
private readonly DataParsingState state = state;
private readonly DataParserFactory dataParserFactory = dataParserFactory;
private readonly IServiceScopeFactory scopeFactory = scopeFactory;
private readonly IWebHostEnvironment webHostEnvironment = webHostEnvironment;
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
@ -36,7 +38,7 @@ namespace Hcs.WebApp.BackgroundServices
state.InvokeOnOperationStarted(operation.Id, operation.CampaignId, startedAt);
var dataParser = dataParserFactory.CreateDataParser(scope, operation);
var dataParser = dataParserFactory.CreateDataParser(scope, operation, webHostEnvironment);
await dataParser.ParseAsync();
}
catch (Exception e)

2
Hcs.WebApp/Components/Dialogs/StartParsing.razor
View File

@ -77,7 +77,7 @@
root.GetProperty("path").GetString(),
root.GetProperty("fileName").GetString(),
UploaderId,
DateTime.Now);
DateTime.UtcNow);
fileToParseId = fileToParse.Id;
}
catch (Exception e)

2
Hcs.WebApp/Hcs.WebApp.csproj
View File

@ -16,6 +16,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="EFCore.BulkExtensions" Version="8.1.3" />
<PackageReference Include="Microsoft.AspNetCore.Diagnostics.EntityFrameworkCore" Version="8.0.20" />
<PackageReference Include="Microsoft.AspNetCore.HeaderPropagation" Version="8.0.20" />
<PackageReference Include="Microsoft.AspNetCore.Identity.EntityFrameworkCore" Version="8.0.20" />
@ -28,7 +29,6 @@
</PackageReference>
<PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="8.0.7" />
<PackageReference Include="Radzen.Blazor" Version="8.0.4" />
<PackageReference Include="Sylvan.Data" Version="0.2.16" />
<PackageReference Include="Sylvan.Data.Excel" Version="0.5.2" />
</ItemGroup>

6
Hcs.WebApp/Services/FileToParseService.cs
View File

@ -27,6 +27,12 @@ namespace Hcs.WebApp.Services
return fileToParse;
}
public async Task<FileToParse> GetFileToParseByOperationIdAsync(int operationId)
{
using var context = GetNewContext();
return await GetFileToParseByOperationIdAsync(context, operationId);
}
public async Task<FileToParse> GetFileToParseByOperationIdAsync(HcsDbContext context, int operationId)
{
return await context.FilesToParse.SingleAsync(x => x.LastParseOperationId == operationId);

19
Hcs.WebApp/Services/HouseService.cs
View File

@ -1,4 +1,5 @@
using Hcs.WebApp.Data.Hcs;
using EFCore.BulkExtensions;
using Hcs.WebApp.Data.Hcs;
using Microsoft.EntityFrameworkCore;
namespace Hcs.WebApp.Services
@ -10,5 +11,21 @@ namespace Hcs.WebApp.Services
using var context = GetNewContext();
return await context.Houses.ToListAsync();
}
public async Task UpsertHouses(IEnumerable<House> houses)
{
using var context = GetNewContext();
await context.BulkInsertOrUpdateAsync(houses, new BulkConfig()
{
PropertiesToExcludeOnUpdate =
[
nameof(House.ThirdPartyId)
],
UpdateByProperties =
[
nameof(House.HcsId)
]
});
}
}
}