Saphire/quest-reader
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Rewrite generator pipeline

Browse Source
This commit is contained in:
Saphire 2022-02-16 01:38:47 +07:00
parent 48296dd5a8
commit 4a040985ee
Signed by: Saphire
GPG Key ID: B26EB7A1F07044C4
8 changed files with 219 additions and 420 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -1,2 +1,5 @@
/target
/quests
/rust
/obj
/bin

67
DataModel.cs Normal file
View File

@ -0,0 +1,67 @@
namespace QuestReader;
using System.Text.Json.Serialization;
public record ThreadPost
{
public int Id { get; set; }
public string Author { get; set; }
public string Uid { get; set; }
public string RawHtml { get; set; }
public string? File { get; set; }
public string? Filename { get; set; }
public string? Title { get; set; }
public string? Tripcode { get; set; }
public DateTime Date { get; set; }
[JsonIgnore]
public bool IsChapterAnnounce { get; set; } = false;
public ChapterMetadata? Chapter { get; set; }
}
public record Metadata
{
public string Name { get; set; }
public string Author { get; set; }
public Uri AuthorPage { get; set; }
public string? AuthorTwitter { get; set; }
public Uri AssetsBaseUrl { get; set; }
public string SocialPreview { get; set; }
public List<int> Threads { get; set; }
public List<ChapterMetadata> Chapters { get; set; }
public override string ToString() => $"\"{Name}\" by {Author}";
}
public record ChapterMetadata
{
public int Id { get; set; }
public string Name { get; set; }
public string Subtitle { get; set; }
public int Start { get; set; }
public int? Announce { get; set; }
public int End { get; set; }
}
public enum ParamType
{
Invalid,
PostId,
UniqueId,
Username
}
public enum ParamError
{
Invalid,
NoError,
NotFound
}
public class TemplateModel
{
public Metadata Metadata { get; set; }
public DateTime Now { get; set; }
public List<ThreadPost> Posts { get; set; }
public string BaseUrl { get; set; }
public string ToolVersion { get; set; }
}

13
HtmlSafeTemplate.cs
View File

@ -2,8 +2,10 @@ using RazorEngineCore;
namespace QuestReader
{
public class HtmlSafeTemplate : RazorEngineTemplateBase
public class HtmlSafeTemplate<TModel> : RazorEngineTemplateBase
{
public new TModel Model { get; set; }
class RawContent
{
public object Value { get; set; }
@ -14,22 +16,21 @@ namespace QuestReader
}
}
public object Raw(object value)
public static object Raw(object value)
{
return new RawContent(value);
}
public override Task WriteAsync(object obj = null)
public override Task WriteAsync(object? obj = null)
{
object value = obj is RawContent rawContent
var value = obj is not null and RawContent rawContent
? rawContent.Value
: System.Web.HttpUtility.HtmlEncode(obj);
return base.WriteAsync(value);
}
public override Task WriteAttributeValueAsync(string prefix, int prefixOffset, object value, int valueOffset, int valueLength, bool isLiteral)
public override Task WriteAttributeValueAsync(string prefix, int prefixOffset, object? value, int valueOffset, int valueLength, bool isLiteral)
{
value = value is RawContent rawContent
? rawContent.Value

425
Program.cs
View File

@ -1,417 +1,22 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using RazorEngineCore;
namespace QuestReader
{
public record ThreadPost
{
public int Id { get; set; }
public string Author { get; set; }
public string Uid { get; set; }
public string RawHtml { get; set; }
public string? File { get; set; }
public string? Filename { get; set; }
public string? Title { get; set; }
public string? Tripcode { get; set; }
public DateTime Date { get; set; }
using QuestReader.Services;
using System.CommandLine;
[JsonIgnore]
public bool IsChapterAnnounce { get; set; } = false;
public ChapterMetadata? Chapter { get; set; }
}
public record Metadata
var command = new RootCommand
{
public string Name { get; set; }
public string Author { get; set; }
public Uri AuthorPage { get; set; }
public string? AuthorTwitter { get; set; }
public Uri AssetsBaseUrl { get; set; }
public string SocialPreview { get; set; }
public List<int> Threads { get; set; }
public List<ChapterMetadata> Chapters { get; set; }
public override string ToString() => $"\"{Name}\" by {Author}";
}
public record ChapterMetadata
{
public int Id { get; set; }
public string Name { get; set; }
public string Subtitle { get; set; }
public int Start { get; set; }
public int? Announce { get; set; }
public int End { get; set; }
}
public enum ParamType
{
Invalid,
PostId,
UniqueId,
Username
}
public enum ParamError
{
Invalid,
NoError,
NotFound
}
public class Program
{
public static async Task Main(string[] args)
{
var options = new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
Name = "quest-parser",
Description = "A tool to archive quest threads",
TreatUnmatchedTokensAsErrors = true
};
if (args.Length < 1) {
Console.WriteLine("Missing quest name, provide it as the first argument");
}
var basePath = $"quests/{args[0]}";
using var fileStream = File.OpenRead(Path.Combine(basePath, "metadata.json"));
var metadata = JsonSerializer.Deserialize<Metadata>(fileStream, options)
?? throw new InvalidDataException("Empty deserialisation result for quest metadata");
fileStream.Dispose();
Console.Out.WriteLine($"Loaded metadata: {metadata}");
var allPosts = metadata.Threads.SelectMany(tId => {
using var fileStream = File.OpenRead(Path.Combine(basePath, $"thread_{tId}.json"));
var threadData = JsonSerializer.Deserialize<List<ThreadPost>>(fileStream, options)
?? throw new InvalidDataException("Empty deserialisation result for thread data");
fileStream.Dispose();
return threadData;
});
var byAuthor = allPosts.GroupBy(p => p.Author).OrderBy(g => g.Count()).ToList();
if (args.Length < 1 || args[1] != "skip") {
for (int i = 0; i < byAuthor.Count; i++)
{
var item = byAuthor[i];
Console.Out.WriteLine($"[{byAuthor.Count - i, 2}] User \"{item.Key}\" has {item.Count()} posts with {item.Where(p => p.File is not null).Count()} files");
}
Console.Write("\nType #s of the author entry (split by ,) or <skip> to load a saved list: ");
}
var inputRaw = args.Length >= 1 && args[1] == "skip" ? "skip" : Console.ReadLine() ?? throw new Exception("AAAA");
var accepted = new List<ThreadPost>();
if (inputRaw != "skip") {
var authors = inputRaw.Split(",").Select(i => byAuthor[^(int.Parse(i.Trim()))].Key);
accepted = ProcessLoop(allPosts, authors);
if (!accepted.Any()) {
Console.Out.WriteLine("\nNothing accepted, nothing written. Exiting");
return;
}
Console.Out.WriteLine("");
using var writer = File.OpenWrite(Path.Join(basePath, $"accepted.json"));
JsonSerializer.Serialize(writer, accepted.Select(i => i.Id).OrderBy(i => i).ToList(), options);
writer.Dispose();
Console.Out.WriteLine($"Successfully wrote {accepted.Count} ids, referencing {accepted.Where(a => a.File is not null).Count()} files");
} else {
using var postsListStream = File.OpenRead(Path.Combine(basePath, "accepted.json"));
var ids = JsonSerializer.Deserialize<List<int>>(postsListStream, options)
?? throw new InvalidDataException("Empty deserialisation result for quest metadata");
accepted = allPosts.Where(p => ids.Contains(p.Id)).ToList();
Console.Out.WriteLine($"Loaded a list of {accepted.Count} posts, referencing {accepted.Where(a => a.File is not null).Count()} files");
}
Console.Write("\nContinue? [yes/NO]: ");
if ((args.Length < 1 || args[1] != "skip") && Console.ReadLine() != "yes")
return;
Console.Out.WriteLine("");
if (!Directory.Exists(Path.Join(basePath, "assets")))
Directory.CreateDirectory(Path.Join(basePath, "assets"));
var files = accepted
.Where(a => a.File is not null)
.Select(a => (a.File, a.Filename))
.ToList();
var downloadTasks = new List<Task>();
using (var client = new HttpClient()) {
client.BaseAddress = metadata.AssetsBaseUrl;
Console.Out.WriteLine($"Downloading missing files...");
await Task.WhenAll(
files
.Where(file => !File.Exists(Path.Join(basePath, "assets", file.File)))
.Select(file => client.GetStreamAsync(file.File).ContinueWith(async (stream) => {
Console.Out.WriteLine($"Downloading {file.File} (aka \"{file.Filename}\")");
using var fileWrite = File.OpenWrite(Path.Join(basePath, "assets", file.File));
await (await stream).CopyToAsync(fileWrite);
})
)
var questNameArg = new Argument<string>(
"questName",
"Quest name to use for loading files and generating"
);
Console.Out.WriteLine($"All files done");
}
command.AddArgument(questNameArg);
var chapterAnnounces = metadata.Chapters.Select(c => c.Announce ?? c.Start);
accepted.Where(p => chapterAnnounces.Contains(p.Id)).ToList().ForEach(p => {
p.IsChapterAnnounce = true;
p.Chapter = metadata.Chapters.Single(c => (c.Announce ?? c.Start) == p.Id);
});
command.SetHandler((string questName) => {
var generator = new Generator(questName);
generator.Run();
}, questNameArg);
var razorEngine = new RazorEngine();
var templateFile = "page_template.cshtml";
var baseUrl = "";
var template = razorEngine.Compile<HtmlSafeTemplate>(File.ReadAllText("page_template.cshtml"));
Console.WriteLine($"Using \"{templateFile}\" with base URL {baseUrl}");
string result = template.Run(instance =>
{
instance.Model = new AnonymousTypeWrapper(new
{
Metadata = metadata,
Posts = accepted,
Now = @DateTime.UtcNow,
//BaseUrl = "assets"
BaseUrl = $"/static/{args[0]}"
});
});
Console.WriteLine($"Template output {result.Length} bytes");
File.WriteAllText(Path.Join(basePath, "output.html"), result);
}
public static List<ThreadPost> ProcessLoop(IEnumerable<ThreadPost> allPosts, IEnumerable<string> authors)
{
/*
Add to processing queue:
q/queue #uid
q/queue @name
a/accept #uid
a/accept @name
a/accept #uid >### - accept with post count more than #
i/inspect #uid
i/inspect 12345
i/inspect @name
Drop _only from queue_
d/del/delete/drop #uid
d/del/delete/drop name
Remove from accepted
r/rm/remove #uid
r/rm/remove name
Status/list:
s/l
l uid(s)
l user(s)/name(s)
l accepted
q/quit
h/help this
*/
var accepted = new List<ThreadPost>();
var processing = allPosts.Where(p => authors.Contains(p.Author)).ToHashSet();
Console.WriteLine($"\nProcesing {processing.Count} posts");
{
var grouped = processing.GroupBy(p => new { p.Uid, p.Author }).OrderBy(g => g.Count());
foreach (var item in grouped)
{
Console.Out.WriteLine($"#{item.Key.Uid} - \"{item.Key.Author}\": {item.Count()} posts");
}
}
var loopedWhenEmpty = false;
while (!loopedWhenEmpty || processing.Any()) {
loopedWhenEmpty = !processing.Any();
if (loopedWhenEmpty) {
loopedWhenEmpty = true;
Console.Out.WriteLine("NOTE: No more entries left in queue. This is the last loop unless you run a <list>, <inspect>, or <add> new entries!");
}
Console.Write("> ");
var loopInput = Console.ReadLine();
if (loopInput is null)
break;
loopInput = loopInput.Trim();
if (loopInput == "quit") {
break;
}
var noParam = !loopInput.Contains(' ', StringComparison.InvariantCulture);
var paramType = ParamType.Invalid;
var paramError = ParamError.NoError;
if (loopInput.StartsWith("i ") || loopInput.StartsWith("inspect ")) {
loopedWhenEmpty = false;
var affected = HandleParams(processing, loopInput, out paramType, out paramError);
if (CheckError(paramType, paramError))
continue;
Console.Out.WriteLine($"{affected.Count} items");
foreach (var item in affected) {
var fileDesc = item.File is not null ? $" - File {item.Filename}" : "";
Console.Out.WriteLine($"{item.Id} - {item.Date: s} - {item.Author} - #{item.Uid}{fileDesc}\n\n{item.RawHtml}\n---");
}
continue;
}
if (loopInput.StartsWith("a ") || loopInput.StartsWith("add ") || loopInput.StartsWith("accept ")) {
var affected = new List<ThreadPost>();
if (loopInput == "a all" || loopInput == "add all" || loopInput == "accept all")
affected = processing.ToList();
else
affected = HandleParams(processing, loopInput, out paramType, out paramError);
if (CheckError(paramType, paramError))
continue;
Console.Out.WriteLine($"{affected.Count} items");
processing.RemoveWhere(i => affected.Contains(i));
accepted.AddRange(affected);
continue;
}
if (loopInput.StartsWith("d ") || loopInput.StartsWith("del ") || loopInput.StartsWith("delete ") || loopInput.StartsWith("drop ")) {
var affected = HandleParams(processing, loopInput, out paramType, out paramError);
if (CheckError(paramType, paramError))
continue;
Console.Out.WriteLine($"{affected.Count} items");
processing.RemoveWhere(i => affected.Contains(i));
continue;
}
if (loopInput.StartsWith("q ") || loopInput.StartsWith("queue ")) {
loopedWhenEmpty = false;
var affected = new List<ThreadPost>();
if (loopInput == "q #" || loopInput == "queue #")
affected = allPosts.Where(p => processing.Any(pr => pr.Uid == p.Uid)).ToList();
else
affected = HandleParams(allPosts, loopInput, out paramType, out paramError);
if (CheckError(paramType, paramError))
continue;
Console.Out.WriteLine($"{affected.Count} items");
processing.UnionWith(affected);
continue;
}
if (noParam && (loopInput.StartsWith("l") || loopInput.StartsWith("list"))) {
var grouped = processing.GroupBy(p => new { p.Uid, p.Author }).OrderBy(g => g.Count());
foreach (var item in grouped)
{
Console.Out.WriteLine($"#{item.Key.Uid} - \"{item.Key.Author}\": {item.Count()} posts");
}
continue;
}
if (loopInput.StartsWith("l ") || loopInput.StartsWith("list ")) {
var split = loopInput.Split(" ");
switch (split[1])
{
case "uid":
case "uids": {
var grouped = allPosts.GroupBy(p => new { p.Uid }).OrderBy(g => g.Count());
foreach (var item in grouped)
{
Console.Out.WriteLine($"#{item.Key.Uid}: {item.Count()} posts {item.Where(i => i.File is not null).Count()} files");
}
break;
}
case "user":
case "users":
case "author":
case "authors":
case "name":
case "names":
case "username":
case "usernames": {
var grouped = allPosts.GroupBy(p => new { p.Author }).OrderBy(g => g.Count());
foreach (var item in grouped)
{
Console.Out.WriteLine($"\"{item.Key.Author}\": {item.Count()} posts with {item.Where(i => i.File is not null).Count()} files");
}
break;
}
case "a":
case "accepted": {
var grouped = accepted.GroupBy(p => new { p.Author, p.Uid }).OrderBy(g => g.Count());
foreach (var item in grouped)
{
Console.Out.WriteLine($"\"{item.Key.Author}\" - #{item.Key.Uid}: {item.Count()} posts with {item.Where(i => i.File is not null).Count()} files");
}
break;
}
default:
break;
}
continue;
}
}
return accepted;
}
public static bool CheckError(ParamType paramType, ParamError paramError)
{
if (paramError == ParamError.NoError)
return false;
Console.Out.WriteLine($"Error {paramError} for param {paramType}");
return true;
}
public static List<ThreadPost> HandleParams(IEnumerable<ThreadPost> targetList, string input, out ParamType paramType, out ParamError paramError, int skip = 1)
{
var output = new List<ThreadPost>();
var split = input.Split(" ").Skip(skip);
paramType = ParamType.Invalid;
paramError = ParamError.Invalid;
foreach (var param in split)
{
if (param.StartsWith("#")) {
paramType = ParamType.UniqueId;
paramError = ParamError.NoError;
output.AddRange(targetList.Where(p => p.Uid == param[1..]));
if (!output.Any())
paramError = ParamError.NotFound;
} else if (param.StartsWith("@")) {
paramType = ParamType.Username;
paramError = ParamError.NoError;
var name = param[1..];
output.AddRange(targetList.Where(p => p.Author == name));
if (!output.Any())
paramError = ParamError.NotFound;
} else if (int.TryParse(param, out var postId)) {
paramType = ParamType.PostId;
paramError = ParamError.NoError;
output.AddRange(targetList.Where(p => p.Id == postId));
if (!output.Any())
paramError = ParamError.NotFound;
}
if (paramError != ParamError.NoError) {
return null!;
}
}
return output.OrderBy(i => i.Id).ToList();
}
}
}
return command.Invoke(args);

65
Services/Generator.cs Normal file
View File

@ -0,0 +1,65 @@
namespace QuestReader.Services;
using System.Reflection;
using RazorEngineCore;
public class Generator
{
public IRazorEngineCompiledTemplate<HtmlSafeTemplate<TemplateModel>> RazorTemplate { get; set; }
public string QuestName { get; set; }
public PostsSource PostsSource { get; set; }
public string QuestPath { get; set; }
public Generator(string questName)
{
QuestPath = $"quests/{questName}";
QuestName = questName;
PostsSource = new PostsSource(questName, QuestPath);
var chapterAnnounces = PostsSource.Metadata.Chapters.Select(c => c.Announce ?? c.Start);
PostsSource.Accepted.Where(p => chapterAnnounces.Contains(p.Id)).ToList().ForEach(p => {
p.IsChapterAnnounce = true;
p.Chapter = PostsSource.Metadata.Chapters.Single(c => (c.Announce ?? c.Start) == p.Id);
});
var razorEngine = new RazorEngine();
var templateFile = "page_template.cshtml";
var baseUrl = "";
RazorTemplate = razorEngine.Compile<HtmlSafeTemplate<TemplateModel>>(
File.ReadAllText("page_template.cshtml"),
option => {
option.AddAssemblyReferenceByName("System.Collections");
option.AddAssemblyReferenceByName("System.Private.Uri");
}
);
Console.WriteLine($"Using \"{templateFile}\" with base URL {baseUrl}");
}
public string Run()
{
string result = RazorTemplate.Run(instance =>
{
instance.Model = new TemplateModel
{
Metadata = PostsSource.Metadata,
Posts = PostsSource.Accepted,
Now = @DateTime.UtcNow,
//BaseUrl = "assets"
BaseUrl = $"/static/{QuestName}",
ToolVersion = Assembly.GetEntryAssembly()?.GetCustomAttribute<AssemblyInformationalVersionAttribute>()?.InformationalVersion ?? "unknown"
};
});
Console.WriteLine($"Template output {result.Length} bytes");
var outputPath = Path.Join(QuestPath, "output.html");
File.WriteAllText(outputPath, result);
Console.WriteLine($"Wrote output to {outputPath}");
return outputPath;
}
}

43
Services/PostsSource.cs Normal file
View File

@ -0,0 +1,43 @@
namespace QuestReader.Services;
using System.Text.Json;
using System.Text.Json.Serialization;
public class PostsSource
{
public List<ThreadPost> Posts { get; set; }
public List<ThreadPost> Accepted { get; set; }
public Metadata Metadata { get; set; }
public PostsSource(string questName, string basePath)
{
var options = new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
};
using var fileStream = File.OpenRead(Path.Combine(basePath, "metadata.json"));
Metadata = JsonSerializer.Deserialize<Metadata>(fileStream, options)
?? throw new InvalidDataException("Empty deserialisation result for quest metadata");
fileStream.Dispose();
Console.Out.WriteLine($"Loaded metadata: {Metadata}");
Posts = Metadata.Threads.SelectMany(tId => {
using var fileStream = File.OpenRead(Path.Combine(basePath, $"thread_{tId}.json"));
var threadData = JsonSerializer.Deserialize<List<ThreadPost>>(fileStream, options)
?? throw new InvalidDataException("Empty deserialisation result for thread data");
fileStream.Dispose();
return threadData;
}).ToList();
using var postsListStream = File.OpenRead(Path.Combine(basePath, "accepted.json"));
var ids = JsonSerializer.Deserialize<List<int>>(postsListStream, options)
?? throw new InvalidDataException("Empty deserialisation result for quest metadata");
Accepted = Posts.Where(p => ids.Contains(p.Id)).ToList();
Console.Out.WriteLine($"Loaded a list of {Accepted.Count} posts, referencing {Accepted.Where(a => a.File is not null).Count()} files");
}
}

7
page_template.cshtml
View File

@ -1,3 +1,7 @@
@namespace QuestReader
@inherits HtmlSafeTemplate<TemplateModel>
<!DOCTYPE html>
<html lang="en">
<head>
@ -35,6 +39,7 @@
<!-- It's just Plausible Analytics - based on ip/etc and daily seed, no incoming requests log. Please email me if you think I am in violation of GDPR/etc -->
<script async defer data-domain="media.lunar.exchange" src="/js/aluminum.js"></script>
<script async defer src="main.js"></script>
</head>
<body>
<header>
@ -69,7 +74,7 @@
}
</main>
<footer>
<p>Ⓒ @Model.Metadata.Author @Model.Posts[0].Date.Year - @Model.Posts[Model.Posts.Count -1].Date.Year, page generated with quest_reader by <a href="https://saphi.re">Saphire</a></p>
<p>Ⓒ @Model.Metadata.Author @Model.Posts[0].Date.Year - @Model.Posts[Model.Posts.Count -1].Date.Year, page generated with <span><a href="https://source.lunar.exchange/Saphire/quest-reader">quest-reader</a> v.<span class="commit-hash">@Model.ToolVersion</span></span> by <a href="https://saphi.re">Saphire</a></p>
</footer>
</body>
</html>

10
quest_reader.csproj
View File

@ -13,5 +13,15 @@
<ItemGroup>
<PackageReference Include="HtmlAgilityPack" Version="1.11.42" />
<PackageReference Include="RazorEngineCore" Version="2022.1.2" />
<PackageReference Include="System.CommandLine" Version="2.0.0-beta3.22111.2" />
</ItemGroup>
<Target Name="SetSourceRevisionId" BeforeTargets="InitializeSourceControlInformation">
<Exec
Command="git describe --long --always --dirty --exclude=* --abbrev=8"
ConsoleToMSBuild="True"
IgnoreExitCode="False"
>
<Output PropertyName="SourceRevisionId" TaskParameter="ConsoleOutput"/>
</Exec>
</Target>
</Project>