Thinking about shunting most of the file parsing into db services since it should be done once

This commit is contained in:
Lani Aung
2021-10-27 07:58:18 -06:00
parent 804ccd0eeb
commit d52cc4b206
16 changed files with 271 additions and 113 deletions
+6 -19
View File
@@ -1,8 +1,8 @@
using System; using System;
using System.Collections.Generic;
using System.Diagnostics; using System.Diagnostics;
using System.IO; using System.IO;
using System.IO.IsolatedStorage; using System.IO.IsolatedStorage;
using System.Linq;
using System.Threading.Tasks; using System.Threading.Tasks;
using fxl.codes.kisekae.Models; using fxl.codes.kisekae.Models;
using fxl.codes.kisekae.Services; using fxl.codes.kisekae.Services;
@@ -29,33 +29,20 @@ namespace fxl.codes.kisekae.Controllers
} }
[HttpGet] [HttpGet]
public IActionResult Index() public async Task<IActionResult> Index()
{ {
var directories = _storage.GetDirectoryNames(); var files = await _databaseService.GetAll();
var model = new List<DirectoryModel>(); return View(files.Select(x => new ConfigurationModel(x)));
foreach (var directory in directories)
{
var files = _storage.GetFileNames(Path.Combine(directory, "*.cnf"));
if (files.Length <= 0) continue;
var doll = new DirectoryModel(directory);
foreach (var file in files) doll.Configurations.Add(new ConfigurationModel(file));
model.Add(doll);
}
return View(model);
} }
[HttpPost] [HttpPost]
public async Task<IActionResult> Upload(IFormFile file) public IActionResult Upload(IFormFile file)
{ {
_logger.LogTrace($"File uploaded: {file?.FileName}"); _logger.LogTrace($"File uploaded: {file?.FileName}");
if (!(file?.FileName.EndsWith("lzh", StringComparison.InvariantCultureIgnoreCase) ?? false)) if (!(file?.FileName.EndsWith("lzh", StringComparison.InvariantCultureIgnoreCase) ?? false))
throw new Exception("Please select a *.lzh file"); throw new Exception("Please select a *.lzh file");
await _databaseService.StoreToDatabase(file); _databaseService.StoreToDatabase(file);
return Redirect("/"); return Redirect("/");
} }
+23
View File
@@ -0,0 +1,23 @@
using System.Linq;
using System.Threading.Tasks;
using fxl.codes.kisekae.Models;
using fxl.codes.kisekae.Services;
using Microsoft.AspNetCore.Mvc;
namespace fxl.codes.kisekae.Controllers
{
public class PlayController : Controller
{
private readonly DatabaseService _databaseService;
public PlayController(DatabaseService databaseService)
{
_databaseService = databaseService;
}
public async Task<IActionResult> Index(int id, int configId)
{
return View();
}
}
}
+10
View File
@@ -0,0 +1,10 @@
using Dapper.Contrib.Extensions;
namespace fxl.codes.kisekae.Entities
{
[Table("cel_config")]
public class CelConfigDto
{
}
}
+3
View File
@@ -1,5 +1,8 @@
using Dapper.Contrib.Extensions;
namespace fxl.codes.kisekae.Entities namespace fxl.codes.kisekae.Entities
{ {
[Table("cel")]
public class CelDto : IKisekaeFile, IKisekaeParseable public class CelDto : IKisekaeFile, IKisekaeParseable
{ {
public int Id { get; set; } public int Id { get; set; }
+9 -1
View File
@@ -1,10 +1,18 @@
using System.Collections.Generic;
using Dapper.Contrib.Extensions;
namespace fxl.codes.kisekae.Entities namespace fxl.codes.kisekae.Entities
{ {
[Table("configuration")]
public class ConfigurationDto : IKisekaeFile public class ConfigurationDto : IKisekaeFile
{ {
public int Id { get; set; } public int Id { get; set; }
public int KisekaeId { get; set; }
public string Filename { get; set; } public string Filename { get; set; }
public string Data { get; set; } public string Data { get; set; }
public int KisekaeId { get; set; } public int? Height { get; set; }
public int? Width { get; set; }
public int? BorderIndex { get; set; }
public IEnumerable<CelConfigDto> Cels { get; set; }
} }
} }
+3 -2
View File
@@ -1,14 +1,15 @@
using System.Collections.Generic; using System.Collections.Generic;
using Dapper.Contrib.Extensions;
namespace fxl.codes.kisekae.Entities namespace fxl.codes.kisekae.Entities
{ {
[Table("kisekae")]
public class KisekaeDto public class KisekaeDto
{ {
public int Id { get; set; } public int Id { get; set; }
public string Name { get; set; } public string Name { get; set; }
public string Filename { get; set; } public string Filename { get; set; }
public string Checksum { get; set; } public string Checksum { get; set; }
public IEnumerable<ConfigurationDto> Configurations { get; set; }
public IEnumerable<CelDto> Cels { get; set; }
} }
} }
+3
View File
@@ -1,5 +1,8 @@
using Dapper.Contrib.Extensions;
namespace fxl.codes.kisekae.Entities namespace fxl.codes.kisekae.Entities
{ {
[Table("palette")]
public class PaletteDto : IKisekaeFile, IKisekaeParseable public class PaletteDto : IKisekaeFile, IKisekaeParseable
{ {
public int Id { get; set; } public int Id { get; set; }
+21
View File
@@ -0,0 +1,21 @@
using System.Collections.Generic;
using fxl.codes.kisekae.Entities;
namespace fxl.codes.kisekae.Models
{
public class ConfigurationModel
{
public ConfigurationModel(KisekaeDto dto)
{
Id = dto.Id;
Name = dto.Name;
Configurations = new Dictionary<int, string>();
foreach (var config in dto.Configurations) Configurations.Add(config.Id, config.Filename);
}
public int Id { get; }
public string Name { get; }
public IDictionary<int, string> Configurations { get; }
}
}
-26
View File
@@ -1,26 +0,0 @@
using System.Collections.Generic;
namespace fxl.codes.kisekae.Models
{
public class DirectoryModel
{
public string Name { get; }
public DirectoryModel(string name)
{
Name = name;
}
public List<ConfigurationModel> Configurations { get; } = new();
}
public class ConfigurationModel
{
public string Name { get; }
public ConfigurationModel(string name)
{
Name = name;
}
}
}
+1
View File
@@ -10,6 +10,7 @@ namespace fxl.codes.kisekae.Models
public string Name { get; set; } public string Name { get; set; }
public int Height { get; set; } public int Height { get; set; }
public int Width { get; set; } public int Width { get; set; }
[JsonIgnore] public int BorderColorIndex { get; set; }
[JsonIgnore] public Color BorderColor { get; set; } = Color.Black; [JsonIgnore] public Color BorderColor { get; set; } = Color.Black;
[JsonIgnore] public List<PaletteModel> Palettes { get; } = new(); [JsonIgnore] public List<PaletteModel> Palettes { get; } = new();
public List<CelModel> Cels { get; } = new(); public List<CelModel> Cels { get; } = new();
+36 -29
View File
@@ -10,6 +10,7 @@ namespace fxl.codes.kisekae.Services
{ {
public class ConfigurationReaderService public class ConfigurationReaderService
{ {
public const string ResolutionRegexPattern = @"\((?<Width>[0-9]*).(?<Height>[0-9]*)\)";
private readonly FileParserService _fileParser; private readonly FileParserService _fileParser;
private readonly ILogger<ConfigurationReaderService> _logger; private readonly ILogger<ConfigurationReaderService> _logger;
@@ -26,36 +27,9 @@ namespace fxl.codes.kisekae.Services
var borderColorIndex = 0; var borderColorIndex = 0;
using var reader = new StreamReader(fileStream); using var reader = new StreamReader(fileStream);
while (!reader.EndOfStream) var data = reader.ReadToEnd();
{
var line = reader.ReadLine();
if (string.IsNullOrEmpty(line)) continue;
switch (line.ToCharArray()[0]) SetPlaysetProperties(data, model, initialPositions);
{
case '(':
var resolutionRegex = new Regex(@"\((?<Width>[0-9]*).(?<Height>[0-9]*)\)");
var resolutionMatch = resolutionRegex.Match(line);
model.Width = int.Parse(resolutionMatch.Groups["Width"].Value);
model.Height = int.Parse(resolutionMatch.Groups["Height"].Value);
break;
case '[':
var borderValue = line.Replace("[", "");
if (borderValue.Contains(';')) borderValue = borderValue.Split(';')[0].Trim();
borderColorIndex = int.Parse(borderValue);
break;
case '%':
model.Palettes.Add(new PaletteModel(_logger, line));
break;
case '#':
model.Cels.Add(new CelModel(_logger, line));
break;
case '$':
case ' ':
initialPositions.Append(line.Replace("\\r\\n", "").Replace("\\n", ""));
break;
}
}
SetInitialPositions(model, initialPositions.ToString()); SetInitialPositions(model, initialPositions.ToString());
if (string.IsNullOrEmpty(directory)) return model; if (string.IsNullOrEmpty(directory)) return model;
@@ -75,6 +49,39 @@ namespace fxl.codes.kisekae.Services
return model; return model;
} }
public void SetPlaysetProperties(string data, PlaysetModel model, StringBuilder initialPositions)
{
if (string.IsNullOrEmpty(data)) return;
foreach (var line in data.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries))
{
switch (line.ToCharArray()[0])
{
case '(':
var resolutionRegex = new Regex(ResolutionRegexPattern);
var resolutionMatch = resolutionRegex.Match(line);
model.Width = int.Parse(resolutionMatch.Groups["Width"].Value);
model.Height = int.Parse(resolutionMatch.Groups["Height"].Value);
break;
case '[':
var borderValue = line[1..];
if (borderValue.Contains(';')) borderValue = borderValue.Split(';')[0].Trim();
model.BorderColorIndex = int.Parse(borderValue);
break;
case '%':
model.Palettes.Add(new PaletteModel(_logger, line));
break;
case '#':
model.Cels.Add(new CelModel(_logger, line));
break;
case '$':
case ' ':
initialPositions.Append(line.Replace("\\r\\n", "").Replace("\\n", ""));
break;
}
}
}
private static void SetInitialPositions(PlaysetModel model, string positions) private static void SetInitialPositions(PlaysetModel model, string positions)
{ {
var sets = positions.Split('$', StringSplitOptions.RemoveEmptyEntries); var sets = positions.Split('$', StringSplitOptions.RemoveEmptyEntries);
+112 -29
View File
@@ -1,10 +1,15 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Data;
using System.IO; using System.IO;
using System.IO.IsolatedStorage; using System.IO.IsolatedStorage;
using System.Linq;
using System.Security.Cryptography; using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks; using System.Threading.Tasks;
using Dapper; using Dapper;
using Dapper.Contrib.Extensions;
using fxl.codes.kisekae.Entities; using fxl.codes.kisekae.Entities;
using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Configuration;
@@ -15,47 +20,96 @@ namespace fxl.codes.kisekae.Services
{ {
public class DatabaseService public class DatabaseService
{ {
private const string ResolutionRegexPattern = @"\((?<Width>[0-9]*).(?<Height>[0-9]*)\)";
private readonly ConfigurationReaderService _configurationReaderService;
private readonly string _connectionString; private readonly string _connectionString;
private readonly FileParserService _fileParserService; private readonly FileParserService _fileParserService;
private readonly ILogger<DatabaseService> _logger; private readonly ILogger<DatabaseService> _logger;
private readonly IsolatedStorageFile _storage; private readonly IsolatedStorageFile _storage;
public DatabaseService(ILogger<DatabaseService> logger, IConfiguration configuration, FileParserService fileParserService) public DatabaseService(ILogger<DatabaseService> logger,
IConfiguration configuration,
ConfigurationReaderService configurationReaderService,
FileParserService fileParserService)
{ {
_logger = logger; _logger = logger;
_configurationReaderService = configurationReaderService;
_fileParserService = fileParserService; _fileParserService = fileParserService;
_connectionString = configuration.GetConnectionString("kisekae"); _connectionString = configuration.GetConnectionString("kisekae");
_storage = IsolatedStorageFile.GetUserStoreForApplication(); _storage = IsolatedStorageFile.GetUserStoreForApplication();
} }
public async Task<KisekaeDto> StoreToDatabase(IFormFile file) public async Task<IEnumerable<KisekaeDto>> GetAll()
{ {
await using var connection = new NpgsqlConnection(_connectionString); await using var connection = new NpgsqlConnection(_connectionString);
await connection.OpenAsync(); await connection.OpenAsync();
var existing = connection.QuerySingleOrDefault<KisekaeDto>("select * from kisekae where kisekae.filename = @Filename", new { Filename = file.FileName }); using var multi = await connection.QueryMultipleAsync("select * from kisekae; select * from configuration");
if (existing != null) var files = multi.Read<KisekaeDto>();
var configs = multi.Read<ConfigurationDto>();
var dictionary = configs.ToLookup(x => x.KisekaeId);
await connection.CloseAsync();
foreach (var kiss in files) kiss.Configurations = dictionary[kiss.Id];
return files;
}
public async void GetKisekaeConfig(int id, int configId)
{
await using var connection = new NpgsqlConnection(_connectionString);
await connection.OpenAsync();
var queryParams = new { Id = id, ConfigId = configId };
var cels = connection.Query<CelConfigDto>("select * from cel_config where configId = @ConfigId", queryParams);
if (cels == null)
{ {
_logger.LogInformation($"Already existing upload {file.FileName} under id {existing.Id}"); var config = connection.QuerySingle<ConfigurationDto>(
return existing; "select * from configuration where kisekae_id = @Id and id = @ConfigId",
queryParams);
foreach (var line in config.Data.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries))
switch (line.ToCharArray()[0])
{
case '(':
var resolutionRegex = new Regex(ResolutionRegexPattern);
var resolutionMatch = resolutionRegex.Match(line);
config.Width = int.Parse(resolutionMatch.Groups["Width"].Value);
config.Height = int.Parse(resolutionMatch.Groups["Height"].Value);
break;
case '[':
var borderValue = line[1..];
if (borderValue.Contains(';')) borderValue = borderValue.Split(';')[0].Trim();
config.BorderIndex = int.Parse(borderValue);
break;
case '%':
break;
case '#':
break;
case '$':
case ' ':
break;
}
await connection.UpdateAsync(config);
} }
await using var memoryStream = new MemoryStream(); await connection.CloseAsync();
await file.CopyToAsync(memoryStream); }
public async void StoreToDatabase(IFormFile file)
{
await using var connection = new NpgsqlConnection(_connectionString);
await connection.OpenAsync();
var memoryStream = await GetAsMemoryStream(file);
var checksum = Convert.ToBase64String(await new SHA256Managed().ComputeHashAsync(memoryStream)); var checksum = Convert.ToBase64String(await new SHA256Managed().ComputeHashAsync(memoryStream));
existing = connection.QuerySingleOrDefault<KisekaeDto>("select * from kisekae where kisekae.checksum = @Checksum", new { Checksum = checksum }); memoryStream.Position = 0; // Reset for re-read
if (existing != null)
{
_logger.LogInformation($"Already existing upload (checksum: {checksum}) under id {existing.Id} and filename {existing.Filename}");
return existing;
}
memoryStream.Position = 0; var existing = GetExisting(connection, file.FileName, checksum);
if (existing != null) return;
_fileParserService.UnzipLzh(file, memoryStream); _fileParserService.UnzipLzh(file, memoryStream);
var directory = Path.GetFileNameWithoutExtension(file.FileName); var directory = Path.GetFileNameWithoutExtension(file.FileName);
var kisekae = new KisekaeDto var kisekae = new KisekaeDto
{ {
Filename = file.FileName, Filename = file.FileName,
@@ -63,13 +117,33 @@ namespace fxl.codes.kisekae.Services
Checksum = checksum Checksum = checksum
}; };
var id = connection.QueryFirst<int>("insert into kisekae(name, filename, checksum) values (@Name, @Filename, @Checksum) returning id", kisekae); kisekae.Id = await connection.InsertAsync(kisekae);
kisekae.Id = id;
var filenames = _storage.GetFileNames($"{Path.Combine(directory, "*")}"); var filenames = _storage.GetFileNames($"{Path.Combine(directory, "*")}");
var cels = new List<CelDto>(); SetInnerFiles(directory, filenames, kisekae.Id);
var palettes = new List<PaletteDto>();
var configurations = new List<ConfigurationDto>(); await connection.CloseAsync();
}
private KisekaeDto GetExisting(IDbConnection connection, string filename, string checksum)
{
var existing = connection.QuerySingleOrDefault<KisekaeDto>("select * from kisekae where kisekae.filename = @Filename", new { Filename = filename });
if (existing != null)
{
_logger.LogInformation($"Already existing upload {filename} under id {existing.Id}");
return existing;
}
existing = connection.QuerySingleOrDefault<KisekaeDto>("select * from kisekae where kisekae.checksum = @Checksum", new { Checksum = checksum });
if (existing == null) return null;
_logger.LogInformation($"Already existing upload (checksum: {checksum}) under id {existing.Id} and filename {existing.Filename}");
return existing;
}
private async void SetInnerFiles(string directory, IEnumerable<string> filenames, int id)
{
var files = new List<IKisekaeFile>();
foreach (var filename in filenames) foreach (var filename in filenames)
{ {
await using var reader = _storage.OpenFile(Path.Combine(directory, filename), FileMode.Open); await using var reader = _storage.OpenFile(Path.Combine(directory, filename), FileMode.Open);
@@ -81,7 +155,7 @@ namespace fxl.codes.kisekae.Services
switch (Path.GetExtension(filename).ToLower()) switch (Path.GetExtension(filename).ToLower())
{ {
case ".cel": case ".cel":
cels.Add(new CelDto files.Add(new CelDto
{ {
Filename = filename, Filename = filename,
Data = bytes, Data = bytes,
@@ -89,15 +163,15 @@ namespace fxl.codes.kisekae.Services
}); });
break; break;
case ".cnf": case ".cnf":
configurations.Add(new ConfigurationDto files.Add(new ConfigurationDto
{ {
Filename = filename, Filename = filename,
Data = BitConverter.ToString(bytes), Data = Encoding.ASCII.GetString(bytes),
KisekaeId = id KisekaeId = id
}); });
break; break;
case ".kcf": case ".kcf":
palettes.Add(new PaletteDto files.Add(new PaletteDto
{ {
Filename = filename, Filename = filename,
Data = bytes, Data = bytes,
@@ -107,13 +181,22 @@ namespace fxl.codes.kisekae.Services
} }
} }
await connection.ExecuteAsync("insert into cel (filename, data, kisekae_id) values (@Filename, @Data, @KisekaeId)", cels); await using var connection = new NpgsqlConnection(_connectionString);
await connection.ExecuteAsync("insert into configuration (filename, data, kisekae_id) values (@Filename, @Data, @KisekaeId)", configurations); await connection.OpenAsync();
await connection.ExecuteAsync("insert into palette (filename, data, kisekae_id) values (@Filename, @Data, @KisekaeId)", palettes);
await connection.InsertAsync(files.OfType<CelDto>());
await connection.InsertAsync(files.OfType<ConfigurationDto>());
await connection.InsertAsync(files.OfType<PaletteDto>());
await connection.CloseAsync(); await connection.CloseAsync();
}
return kisekae; private static async Task<MemoryStream> GetAsMemoryStream(IFormFile file)
{
var stream = new MemoryStream();
await file.CopyToAsync(stream);
return stream;
} }
} }
} }
+1 -1
View File
@@ -56,7 +56,7 @@ namespace fxl.codes.kisekae
{ {
endpoints.MapControllerRoute( endpoints.MapControllerRoute(
"default", "default",
"{controller=Home}/{action=Index}/{id?}"); "{controller=Home}/{action=Index}/{id?}/{configId?}");
}); });
DefaultTypeMap.MatchNamesWithUnderscores = true; DefaultTypeMap.MatchNamesWithUnderscores = true;
+5 -5
View File
@@ -1,4 +1,4 @@
@model IEnumerable<DirectoryModel> @model IEnumerable<ConfigurationModel>
@{ @{
ViewData["Title"] = "Select"; ViewData["Title"] = "Select";
@@ -10,15 +10,15 @@
</form> </form>
<ul> <ul>
@foreach (var directory in Model) @foreach (var kiss in Model)
{ {
<li> <li>
@directory.Name @kiss.Name
<ul> <ul>
@foreach (var config in directory.Configurations) @foreach (var (key, value) in kiss.Configurations)
{ {
<li> <li>
<a href="javascript:" data-directory="@directory.Name">@config.Name</a> <a asp-controller="Play" asp-action="Index" asp-route-id="@kiss.Id" asp-route-configId="@key">@value</a>
</li> </li>
} }
</ul> </ul>
+36
View File
@@ -0,0 +1,36 @@
@model PlaysetModel
@{
ViewData["Title"] = $"Play with {Model.Name}";
}
<header id="play_menu">
<ul>
<li>
<button class="mdc-button mdc-button--icon-trailing" data-rel="reset">
<span class="mdc-button__ripple"></span>
<span class="mdc-button__label">Reset</span>
<i class="material-icons mdc-button__icon" aria-hidden="true">restart_alt</i>
</button>
</li>
<li class="mdc-typography--button">Sets</li>
</ul>
</header>
<article id="play_area" style="background-color: #@Model.BorderColor">
<div id="play_space" style="height: @(Model.Height)px; width: @(Model.Width)px">
@for (var index = 0; index < Model.Cels.Count; index++)
{
var cel = Model.Cels[index];
<div class="cel-image"
style="background-image: url('data:image/gif;base64,@cel.DefaultImage'); height: @(cel.Height)px; width: @(cel.Width)px; z-index: @cel.ZIndex; opacity: @cel.Opacity"
data-id="@cel.Id"
data-index="@index"></div>
}
</div>
</article>
@section Scripts
{
<script>
document.kisekae.load("play_space", "play_menu", @Json.Serialize(Model))
</script>
}
+1
View File
@@ -14,6 +14,7 @@
<ItemGroup> <ItemGroup>
<PackageReference Include="Dapper" Version="2.0.90" /> <PackageReference Include="Dapper" Version="2.0.90" />
<PackageReference Include="Dapper.Contrib" Version="2.0.78" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.Razor.RuntimeCompilation" Version="5.0.10" /> <PackageReference Include="Microsoft.AspNetCore.Mvc.Razor.RuntimeCompilation" Version="5.0.10" />
<PackageReference Include="Npgsql" Version="5.0.10" /> <PackageReference Include="Npgsql" Version="5.0.10" />
<PackageReference Include="SixLabors.ImageSharp" Version="1.0.4" /> <PackageReference Include="SixLabors.ImageSharp" Version="1.0.4" />