From 4a040985ee64ef488a8eb80e56b8c94bbc53636d Mon Sep 17 00:00:00 2001 From: Saphire Date: Wed, 16 Feb 2022 01:38:47 +0700 Subject: [PATCH] Rewrite generator pipeline --- .gitignore | 3 + DataModel.cs | 67 +++++++ HtmlSafeTemplate.cs | 13 +- Program.cs | 431 ++-------------------------------------- Services/Generator.cs | 65 ++++++ Services/PostsSource.cs | 43 ++++ page_template.cshtml | 7 +- quest_reader.csproj | 10 + 8 files changed, 219 insertions(+), 420 deletions(-) create mode 100644 DataModel.cs create mode 100644 Services/Generator.cs create mode 100644 Services/PostsSource.cs diff --git a/.gitignore b/.gitignore index fc75352..44d1e0b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ /target /quests +/rust +/obj +/bin \ No newline at end of file diff --git a/DataModel.cs b/DataModel.cs new file mode 100644 index 0000000..57ddc42 --- /dev/null +++ b/DataModel.cs @@ -0,0 +1,67 @@ +namespace QuestReader; + +using System.Text.Json.Serialization; + +public record ThreadPost +{ + public int Id { get; set; } + public string Author { get; set; } + public string Uid { get; set; } + public string RawHtml { get; set; } + public string? File { get; set; } + public string? Filename { get; set; } + public string? Title { get; set; } + public string? Tripcode { get; set; } + public DateTime Date { get; set; } + + [JsonIgnore] + public bool IsChapterAnnounce { get; set; } = false; + public ChapterMetadata? Chapter { get; set; } +} + +public record Metadata +{ + public string Name { get; set; } + public string Author { get; set; } + public Uri AuthorPage { get; set; } + public string? AuthorTwitter { get; set; } + public Uri AssetsBaseUrl { get; set; } + public string SocialPreview { get; set; } + public List Threads { get; set; } + public List Chapters { get; set; } + + public override string ToString() => $"\"{Name}\" by {Author}"; +} +public record ChapterMetadata +{ + public int Id { get; set; } + public string Name { get; set; } + public string Subtitle { get; set; } + public int Start { get; set; } + public int? Announce { get; set; } + public int End { get; set; } +} + +public enum ParamType +{ + Invalid, + PostId, + UniqueId, + Username +} + +public enum ParamError +{ + Invalid, + NoError, + NotFound +} + +public class TemplateModel +{ + public Metadata Metadata { get; set; } + public DateTime Now { get; set; } + public List Posts { get; set; } + public string BaseUrl { get; set; } + public string ToolVersion { get; set; } +} \ No newline at end of file diff --git a/HtmlSafeTemplate.cs b/HtmlSafeTemplate.cs index 6e8c2e9..a7a9e62 100644 --- a/HtmlSafeTemplate.cs +++ b/HtmlSafeTemplate.cs @@ -2,8 +2,10 @@ using RazorEngineCore; namespace QuestReader { - public class HtmlSafeTemplate : RazorEngineTemplateBase + public class HtmlSafeTemplate : RazorEngineTemplateBase { + public new TModel Model { get; set; } + class RawContent { public object Value { get; set; } @@ -14,22 +16,21 @@ namespace QuestReader } } - - public object Raw(object value) + public static object Raw(object value) { return new RawContent(value); } - public override Task WriteAsync(object obj = null) + public override Task WriteAsync(object? obj = null) { - object value = obj is RawContent rawContent + var value = obj is not null and RawContent rawContent ? rawContent.Value : System.Web.HttpUtility.HtmlEncode(obj); return base.WriteAsync(value); } - public override Task WriteAttributeValueAsync(string prefix, int prefixOffset, object value, int valueOffset, int valueLength, bool isLiteral) + public override Task WriteAttributeValueAsync(string prefix, int prefixOffset, object? value, int valueOffset, int valueLength, bool isLiteral) { value = value is RawContent rawContent ? rawContent.Value diff --git a/Program.cs b/Program.cs index e1a3a8a..f30a658 100644 --- a/Program.cs +++ b/Program.cs @@ -1,417 +1,22 @@ -using System.Text.Json; -using System.Text.Json.Serialization; -using RazorEngineCore; -namespace QuestReader +using QuestReader.Services; +using System.CommandLine; + +var command = new RootCommand { - public record ThreadPost - { - public int Id { get; set; } - public string Author { get; set; } - public string Uid { get; set; } - public string RawHtml { get; set; } - public string? File { get; set; } - public string? Filename { get; set; } - public string? Title { get; set; } - public string? Tripcode { get; set; } - public DateTime Date { get; set; } + Name = "quest-parser", + Description = "A tool to archive quest threads", + TreatUnmatchedTokensAsErrors = true +}; +var questNameArg = new Argument( + "questName", + "Quest name to use for loading files and generating" +); +command.AddArgument(questNameArg); - [JsonIgnore] - public bool IsChapterAnnounce { get; set; } = false; - public ChapterMetadata? Chapter { get; set; } - } +command.SetHandler((string questName) => { + var generator = new Generator(questName); + generator.Run(); +}, questNameArg); - public record Metadata - { - public string Name { get; set; } - public string Author { get; set; } - public Uri AuthorPage { get; set; } - public string? AuthorTwitter { get; set; } - public Uri AssetsBaseUrl { get; set; } - public string SocialPreview { get; set; } - public List Threads { get; set; } - public List Chapters { get; set; } - - public override string ToString() => $"\"{Name}\" by {Author}"; - } - public record ChapterMetadata - { - public int Id { get; set; } - public string Name { get; set; } - public string Subtitle { get; set; } - public int Start { get; set; } - public int? Announce { get; set; } - public int End { get; set; } - } - - public enum ParamType - { - Invalid, - PostId, - UniqueId, - Username - } - - public enum ParamError - { - Invalid, - NoError, - NotFound - } - - public class Program - { - - public static async Task Main(string[] args) - { - var options = new JsonSerializerOptions - { - PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull - }; - - if (args.Length < 1) { - Console.WriteLine("Missing quest name, provide it as the first argument"); - } - - var basePath = $"quests/{args[0]}"; - - using var fileStream = File.OpenRead(Path.Combine(basePath, "metadata.json")); - var metadata = JsonSerializer.Deserialize(fileStream, options) - ?? throw new InvalidDataException("Empty deserialisation result for quest metadata"); - fileStream.Dispose(); - - Console.Out.WriteLine($"Loaded metadata: {metadata}"); - var allPosts = metadata.Threads.SelectMany(tId => { - using var fileStream = File.OpenRead(Path.Combine(basePath, $"thread_{tId}.json")); - var threadData = JsonSerializer.Deserialize>(fileStream, options) - ?? throw new InvalidDataException("Empty deserialisation result for thread data"); - fileStream.Dispose(); - - return threadData; - }); - - var byAuthor = allPosts.GroupBy(p => p.Author).OrderBy(g => g.Count()).ToList(); - - if (args.Length < 1 || args[1] != "skip") { - for (int i = 0; i < byAuthor.Count; i++) - { - var item = byAuthor[i]; - Console.Out.WriteLine($"[{byAuthor.Count - i, 2}] User \"{item.Key}\" has {item.Count()} posts with {item.Where(p => p.File is not null).Count()} files"); - } - Console.Write("\nType #s of the author entry (split by ,) or to load a saved list: "); - } - var inputRaw = args.Length >= 1 && args[1] == "skip" ? "skip" : Console.ReadLine() ?? throw new Exception("AAAA"); - var accepted = new List(); - if (inputRaw != "skip") { - var authors = inputRaw.Split(",").Select(i => byAuthor[^(int.Parse(i.Trim()))].Key); - accepted = ProcessLoop(allPosts, authors); - - if (!accepted.Any()) { - Console.Out.WriteLine("\nNothing accepted, nothing written. Exiting"); - return; - } - Console.Out.WriteLine(""); - using var writer = File.OpenWrite(Path.Join(basePath, $"accepted.json")); - JsonSerializer.Serialize(writer, accepted.Select(i => i.Id).OrderBy(i => i).ToList(), options); - writer.Dispose(); - Console.Out.WriteLine($"Successfully wrote {accepted.Count} ids, referencing {accepted.Where(a => a.File is not null).Count()} files"); - } else { - using var postsListStream = File.OpenRead(Path.Combine(basePath, "accepted.json")); - var ids = JsonSerializer.Deserialize>(postsListStream, options) - ?? throw new InvalidDataException("Empty deserialisation result for quest metadata"); - accepted = allPosts.Where(p => ids.Contains(p.Id)).ToList(); - Console.Out.WriteLine($"Loaded a list of {accepted.Count} posts, referencing {accepted.Where(a => a.File is not null).Count()} files"); - } - - Console.Write("\nContinue? [yes/NO]: "); - if ((args.Length < 1 || args[1] != "skip") && Console.ReadLine() != "yes") - return; - - Console.Out.WriteLine(""); - - if (!Directory.Exists(Path.Join(basePath, "assets"))) - Directory.CreateDirectory(Path.Join(basePath, "assets")); - - var files = accepted - .Where(a => a.File is not null) - .Select(a => (a.File, a.Filename)) - .ToList(); - - var downloadTasks = new List(); - - using (var client = new HttpClient()) { - client.BaseAddress = metadata.AssetsBaseUrl; - Console.Out.WriteLine($"Downloading missing files..."); - - await Task.WhenAll( - files - .Where(file => !File.Exists(Path.Join(basePath, "assets", file.File))) - .Select(file => client.GetStreamAsync(file.File).ContinueWith(async (stream) => { - Console.Out.WriteLine($"Downloading {file.File} (aka \"{file.Filename}\")"); - using var fileWrite = File.OpenWrite(Path.Join(basePath, "assets", file.File)); - await (await stream).CopyToAsync(fileWrite); - }) - ) - ); - Console.Out.WriteLine($"All files done"); - } - - var chapterAnnounces = metadata.Chapters.Select(c => c.Announce ?? c.Start); - accepted.Where(p => chapterAnnounces.Contains(p.Id)).ToList().ForEach(p => { - p.IsChapterAnnounce = true; - p.Chapter = metadata.Chapters.Single(c => (c.Announce ?? c.Start) == p.Id); - }); - - var razorEngine = new RazorEngine(); - var templateFile = "page_template.cshtml"; - var baseUrl = ""; - var template = razorEngine.Compile(File.ReadAllText("page_template.cshtml")); - - Console.WriteLine($"Using \"{templateFile}\" with base URL {baseUrl}"); - string result = template.Run(instance => - { - instance.Model = new AnonymousTypeWrapper(new - { - Metadata = metadata, - Posts = accepted, - Now = @DateTime.UtcNow, - //BaseUrl = "assets" - BaseUrl = $"/static/{args[0]}" - }); - }); - - Console.WriteLine($"Template output {result.Length} bytes"); - File.WriteAllText(Path.Join(basePath, "output.html"), result); - } - - public static List ProcessLoop(IEnumerable allPosts, IEnumerable authors) - { - - /* - Add to processing queue: - q/queue #uid - q/queue @name - - a/accept #uid - a/accept @name - a/accept #uid >### - accept with post count more than # - - i/inspect #uid - i/inspect 12345 - i/inspect @name - - Drop _only from queue_ - d/del/delete/drop #uid - d/del/delete/drop name - - Remove from accepted - r/rm/remove #uid - r/rm/remove name - - Status/list: - s/l - l uid(s) - l user(s)/name(s) - l accepted - - q/quit - h/help this - */ - - var accepted = new List(); - var processing = allPosts.Where(p => authors.Contains(p.Author)).ToHashSet(); - - Console.WriteLine($"\nProcesing {processing.Count} posts"); - { - var grouped = processing.GroupBy(p => new { p.Uid, p.Author }).OrderBy(g => g.Count()); - foreach (var item in grouped) - { - Console.Out.WriteLine($"#{item.Key.Uid} - \"{item.Key.Author}\": {item.Count()} posts"); - } - } - - var loopedWhenEmpty = false; - while (!loopedWhenEmpty || processing.Any()) { - loopedWhenEmpty = !processing.Any(); - if (loopedWhenEmpty) { - loopedWhenEmpty = true; - Console.Out.WriteLine("NOTE: No more entries left in queue. This is the last loop unless you run a , , or new entries!"); - } - - Console.Write("> "); - var loopInput = Console.ReadLine(); - if (loopInput is null) - break; - loopInput = loopInput.Trim(); - if (loopInput == "quit") { - break; - } - - var noParam = !loopInput.Contains(' ', StringComparison.InvariantCulture); - var paramType = ParamType.Invalid; - var paramError = ParamError.NoError; - - if (loopInput.StartsWith("i ") || loopInput.StartsWith("inspect ")) { - loopedWhenEmpty = false; - var affected = HandleParams(processing, loopInput, out paramType, out paramError); - if (CheckError(paramType, paramError)) - continue; - - Console.Out.WriteLine($"{affected.Count} items"); - foreach (var item in affected) { - var fileDesc = item.File is not null ? $" - File {item.Filename}" : ""; - Console.Out.WriteLine($"{item.Id} - {item.Date: s} - {item.Author} - #{item.Uid}{fileDesc}\n\n{item.RawHtml}\n---"); - } - continue; - } - - if (loopInput.StartsWith("a ") || loopInput.StartsWith("add ") || loopInput.StartsWith("accept ")) { - var affected = new List(); - if (loopInput == "a all" || loopInput == "add all" || loopInput == "accept all") - affected = processing.ToList(); - else - affected = HandleParams(processing, loopInput, out paramType, out paramError); - if (CheckError(paramType, paramError)) - continue; - - Console.Out.WriteLine($"{affected.Count} items"); - processing.RemoveWhere(i => affected.Contains(i)); - accepted.AddRange(affected); - continue; - } - - if (loopInput.StartsWith("d ") || loopInput.StartsWith("del ") || loopInput.StartsWith("delete ") || loopInput.StartsWith("drop ")) { - var affected = HandleParams(processing, loopInput, out paramType, out paramError); - if (CheckError(paramType, paramError)) - continue; - - Console.Out.WriteLine($"{affected.Count} items"); - processing.RemoveWhere(i => affected.Contains(i)); - continue; - } - - if (loopInput.StartsWith("q ") || loopInput.StartsWith("queue ")) { - loopedWhenEmpty = false; - var affected = new List(); - if (loopInput == "q #" || loopInput == "queue #") - affected = allPosts.Where(p => processing.Any(pr => pr.Uid == p.Uid)).ToList(); - else - affected = HandleParams(allPosts, loopInput, out paramType, out paramError); - - if (CheckError(paramType, paramError)) - continue; - - Console.Out.WriteLine($"{affected.Count} items"); - processing.UnionWith(affected); - continue; - } - - if (noParam && (loopInput.StartsWith("l") || loopInput.StartsWith("list"))) { - var grouped = processing.GroupBy(p => new { p.Uid, p.Author }).OrderBy(g => g.Count()); - foreach (var item in grouped) - { - Console.Out.WriteLine($"#{item.Key.Uid} - \"{item.Key.Author}\": {item.Count()} posts"); - } - continue; - } - - if (loopInput.StartsWith("l ") || loopInput.StartsWith("list ")) { - var split = loopInput.Split(" "); - switch (split[1]) - { - case "uid": - case "uids": { - var grouped = allPosts.GroupBy(p => new { p.Uid }).OrderBy(g => g.Count()); - foreach (var item in grouped) - { - Console.Out.WriteLine($"#{item.Key.Uid}: {item.Count()} posts {item.Where(i => i.File is not null).Count()} files"); - } - break; - } - - case "user": - case "users": - case "author": - case "authors": - case "name": - case "names": - case "username": - case "usernames": { - var grouped = allPosts.GroupBy(p => new { p.Author }).OrderBy(g => g.Count()); - foreach (var item in grouped) - { - Console.Out.WriteLine($"\"{item.Key.Author}\": {item.Count()} posts with {item.Where(i => i.File is not null).Count()} files"); - } - break; - } - - case "a": - case "accepted": { - var grouped = accepted.GroupBy(p => new { p.Author, p.Uid }).OrderBy(g => g.Count()); - foreach (var item in grouped) - { - Console.Out.WriteLine($"\"{item.Key.Author}\" - #{item.Key.Uid}: {item.Count()} posts with {item.Where(i => i.File is not null).Count()} files"); - } - break; - } - - - default: - break; - } - continue; - } - } - - return accepted; - } - - public static bool CheckError(ParamType paramType, ParamError paramError) - { - if (paramError == ParamError.NoError) - return false; - Console.Out.WriteLine($"Error {paramError} for param {paramType}"); - return true; - } - - public static List HandleParams(IEnumerable targetList, string input, out ParamType paramType, out ParamError paramError, int skip = 1) - { - var output = new List(); - var split = input.Split(" ").Skip(skip); - paramType = ParamType.Invalid; - paramError = ParamError.Invalid; - - foreach (var param in split) - { - if (param.StartsWith("#")) { - paramType = ParamType.UniqueId; - paramError = ParamError.NoError; - output.AddRange(targetList.Where(p => p.Uid == param[1..])); - if (!output.Any()) - paramError = ParamError.NotFound; - } else if (param.StartsWith("@")) { - paramType = ParamType.Username; - paramError = ParamError.NoError; - var name = param[1..]; - output.AddRange(targetList.Where(p => p.Author == name)); - if (!output.Any()) - paramError = ParamError.NotFound; - } else if (int.TryParse(param, out var postId)) { - paramType = ParamType.PostId; - paramError = ParamError.NoError; - output.AddRange(targetList.Where(p => p.Id == postId)); - if (!output.Any()) - paramError = ParamError.NotFound; - } - - if (paramError != ParamError.NoError) { - return null!; - } - } - - - return output.OrderBy(i => i.Id).ToList(); - } - } -} \ No newline at end of file +return command.Invoke(args); diff --git a/Services/Generator.cs b/Services/Generator.cs new file mode 100644 index 0000000..de0e2af --- /dev/null +++ b/Services/Generator.cs @@ -0,0 +1,65 @@ +namespace QuestReader.Services; + +using System.Reflection; +using RazorEngineCore; + +public class Generator +{ + public IRazorEngineCompiledTemplate> RazorTemplate { get; set; } + + public string QuestName { get; set; } + + public PostsSource PostsSource { get; set; } + + public string QuestPath { get; set; } + + public Generator(string questName) + { + QuestPath = $"quests/{questName}"; + + QuestName = questName; + PostsSource = new PostsSource(questName, QuestPath); + + var chapterAnnounces = PostsSource.Metadata.Chapters.Select(c => c.Announce ?? c.Start); + + PostsSource.Accepted.Where(p => chapterAnnounces.Contains(p.Id)).ToList().ForEach(p => { + p.IsChapterAnnounce = true; + p.Chapter = PostsSource.Metadata.Chapters.Single(c => (c.Announce ?? c.Start) == p.Id); + }); + + var razorEngine = new RazorEngine(); + var templateFile = "page_template.cshtml"; + var baseUrl = ""; + RazorTemplate = razorEngine.Compile>( + File.ReadAllText("page_template.cshtml"), + option => { + option.AddAssemblyReferenceByName("System.Collections"); + option.AddAssemblyReferenceByName("System.Private.Uri"); + } + ); + + Console.WriteLine($"Using \"{templateFile}\" with base URL {baseUrl}"); + } + + public string Run() + { + string result = RazorTemplate.Run(instance => + { + instance.Model = new TemplateModel + { + Metadata = PostsSource.Metadata, + Posts = PostsSource.Accepted, + Now = @DateTime.UtcNow, + //BaseUrl = "assets" + BaseUrl = $"/static/{QuestName}", + ToolVersion = Assembly.GetEntryAssembly()?.GetCustomAttribute()?.InformationalVersion ?? "unknown" + }; + }); + + Console.WriteLine($"Template output {result.Length} bytes"); + var outputPath = Path.Join(QuestPath, "output.html"); + File.WriteAllText(outputPath, result); + Console.WriteLine($"Wrote output to {outputPath}"); + return outputPath; + } +} diff --git a/Services/PostsSource.cs b/Services/PostsSource.cs new file mode 100644 index 0000000..e6e9727 --- /dev/null +++ b/Services/PostsSource.cs @@ -0,0 +1,43 @@ +namespace QuestReader.Services; + +using System.Text.Json; +using System.Text.Json.Serialization; + +public class PostsSource +{ + public List Posts { get; set; } + + public List Accepted { get; set; } + + public Metadata Metadata { get; set; } + + public PostsSource(string questName, string basePath) + { + var options = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull + }; + + using var fileStream = File.OpenRead(Path.Combine(basePath, "metadata.json")); + Metadata = JsonSerializer.Deserialize(fileStream, options) + ?? throw new InvalidDataException("Empty deserialisation result for quest metadata"); + fileStream.Dispose(); + + Console.Out.WriteLine($"Loaded metadata: {Metadata}"); + Posts = Metadata.Threads.SelectMany(tId => { + using var fileStream = File.OpenRead(Path.Combine(basePath, $"thread_{tId}.json")); + var threadData = JsonSerializer.Deserialize>(fileStream, options) + ?? throw new InvalidDataException("Empty deserialisation result for thread data"); + fileStream.Dispose(); + + return threadData; + }).ToList(); + + using var postsListStream = File.OpenRead(Path.Combine(basePath, "accepted.json")); + var ids = JsonSerializer.Deserialize>(postsListStream, options) + ?? throw new InvalidDataException("Empty deserialisation result for quest metadata"); + Accepted = Posts.Where(p => ids.Contains(p.Id)).ToList(); + Console.Out.WriteLine($"Loaded a list of {Accepted.Count} posts, referencing {Accepted.Where(a => a.File is not null).Count()} files"); + } +} \ No newline at end of file diff --git a/page_template.cshtml b/page_template.cshtml index d9da43e..13dcd52 100644 --- a/page_template.cshtml +++ b/page_template.cshtml @@ -1,3 +1,7 @@ + +@namespace QuestReader +@inherits HtmlSafeTemplate + @@ -35,6 +39,7 @@ +
@@ -69,7 +74,7 @@ }
-

Ⓒ @Model.Metadata.Author @Model.Posts[0].Date.Year - @Model.Posts[Model.Posts.Count -1].Date.Year, page generated with quest_reader by Saphire

+

Ⓒ @Model.Metadata.Author @Model.Posts[0].Date.Year - @Model.Posts[Model.Posts.Count -1].Date.Year, page generated with quest-reader v.@Model.ToolVersion by Saphire

\ No newline at end of file diff --git a/quest_reader.csproj b/quest_reader.csproj index d62ecb8..98c6ee8 100644 --- a/quest_reader.csproj +++ b/quest_reader.csproj @@ -13,5 +13,15 @@ + + + + + +