Implement a basic initial working version

This commit is contained in:
Saphire 2022-02-14 00:25:15 +07:00
commit fa925faf03
Signed by: Saphire
GPG Key ID: B26EB7A1F07044C4
7 changed files with 632 additions and 0 deletions

2
.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
/target
/quests

41
HtmlSafeTemplate.cs Normal file
View File

@ -0,0 +1,41 @@
using RazorEngineCore;
namespace QuestReader
{
public class HtmlSafeTemplate : RazorEngineTemplateBase
{
class RawContent
{
public object Value { get; set; }
public RawContent(object value)
{
Value = value;
}
}
public object Raw(object value)
{
return new RawContent(value);
}
public override Task WriteAsync(object obj = null)
{
object value = obj is RawContent rawContent
? rawContent.Value
: System.Web.HttpUtility.HtmlEncode(obj);
return base.WriteAsync(value);
}
public override Task WriteAttributeValueAsync(string prefix, int prefixOffset, object value, int valueOffset, int valueLength, bool isLiteral)
{
value = value is RawContent rawContent
? rawContent.Value
: System.Web.HttpUtility.HtmlAttributeEncode(value?.ToString());
return base.WriteAttributeValueAsync(prefix, prefixOffset, value, valueOffset, valueLength, isLiteral);
}
}
}

417
Program.cs Normal file
View File

@ -0,0 +1,417 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using RazorEngineCore;
namespace QuestReader
{
public record ThreadPost
{
public int Id { get; set; }
public string Author { get; set; }
public string Uid { get; set; }
public string RawHtml { get; set; }
public string? File { get; set; }
public string? Filename { get; set; }
public string? Title { get; set; }
public string? Tripcode { get; set; }
public DateTime Date { get; set; }
[JsonIgnore]
public bool IsChapterAnnounce { get; set; } = false;
public ChapterMetadata? Chapter { get; set; }
}
public record Metadata
{
public string Name { get; set; }
public string Author { get; set; }
public Uri AuthorPage { get; set; }
public string? AuthorTwitter { get; set; }
public Uri AssetsBaseUrl { get; set; }
public string SocialPreview { get; set; }
public List<int> Threads { get; set; }
public List<ChapterMetadata> Chapters { get; set; }
public override string ToString() => $"\"{Name}\" by {Author}";
}
public record ChapterMetadata
{
public int Id { get; set; }
public string Name { get; set; }
public string Subtitle { get; set; }
public int Start { get; set; }
public int? Announce { get; set; }
public int End { get; set; }
}
public enum ParamType
{
Invalid,
PostId,
UniqueId,
Username
}
public enum ParamError
{
Invalid,
NoError,
NotFound
}
public class Program
{
public static async Task Main(string[] args)
{
var options = new JsonSerializerOptions
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
};
if (args.Length < 1) {
Console.WriteLine("Missing quest name, provide it as the first argument");
}
var basePath = $"quests/{args[0]}";
using var fileStream = File.OpenRead(Path.Combine(basePath, "metadata.json"));
var metadata = JsonSerializer.Deserialize<Metadata>(fileStream, options)
?? throw new InvalidDataException("Empty deserialisation result for quest metadata");
fileStream.Dispose();
Console.Out.WriteLine($"Loaded metadata: {metadata}");
var allPosts = metadata.Threads.SelectMany(tId => {
using var fileStream = File.OpenRead(Path.Combine(basePath, $"thread_{tId}.json"));
var threadData = JsonSerializer.Deserialize<List<ThreadPost>>(fileStream, options)
?? throw new InvalidDataException("Empty deserialisation result for thread data");
fileStream.Dispose();
return threadData;
});
var byAuthor = allPosts.GroupBy(p => p.Author).OrderBy(g => g.Count()).ToList();
if (args.Length < 1 || args[1] != "skip") {
for (int i = 0; i < byAuthor.Count; i++)
{
var item = byAuthor[i];
Console.Out.WriteLine($"[{byAuthor.Count - i, 2}] User \"{item.Key}\" has {item.Count()} posts with {item.Where(p => p.File is not null).Count()} files");
}
Console.Write("\nType #s of the author entry (split by ,) or <skip> to load a saved list: ");
}
var inputRaw = args.Length >= 1 && args[1] == "skip" ? "skip" : Console.ReadLine() ?? throw new Exception("AAAA");
var accepted = new List<ThreadPost>();
if (inputRaw != "skip") {
var authors = inputRaw.Split(",").Select(i => byAuthor[^(int.Parse(i.Trim()))].Key);
accepted = ProcessLoop(allPosts, authors);
if (!accepted.Any()) {
Console.Out.WriteLine("\nNothing accepted, nothing written. Exiting");
return;
}
Console.Out.WriteLine("");
using var writer = File.OpenWrite(Path.Join(basePath, $"accepted.json"));
JsonSerializer.Serialize(writer, accepted.Select(i => i.Id).OrderBy(i => i).ToList(), options);
writer.Dispose();
Console.Out.WriteLine($"Successfully wrote {accepted.Count} ids, referencing {accepted.Where(a => a.File is not null).Count()} files");
} else {
using var postsListStream = File.OpenRead(Path.Combine(basePath, "accepted.json"));
var ids = JsonSerializer.Deserialize<List<int>>(postsListStream, options)
?? throw new InvalidDataException("Empty deserialisation result for quest metadata");
accepted = allPosts.Where(p => ids.Contains(p.Id)).ToList();
Console.Out.WriteLine($"Loaded a list of {accepted.Count} posts, referencing {accepted.Where(a => a.File is not null).Count()} files");
}
Console.Write("\nContinue? [yes/NO]: ");
if ((args.Length < 1 || args[1] != "skip") && Console.ReadLine() != "yes")
return;
Console.Out.WriteLine("");
if (!Directory.Exists(Path.Join(basePath, "assets")))
Directory.CreateDirectory(Path.Join(basePath, "assets"));
var files = accepted
.Where(a => a.File is not null)
.Select(a => (a.File, a.Filename))
.ToList();
var downloadTasks = new List<Task>();
using (var client = new HttpClient()) {
client.BaseAddress = metadata.AssetsBaseUrl;
Console.Out.WriteLine($"Downloading missing files...");
await Task.WhenAll(
files
.Where(file => !File.Exists(Path.Join(basePath, "assets", file.File)))
.Select(file => client.GetStreamAsync(file.File).ContinueWith(async (stream) => {
Console.Out.WriteLine($"Downloading {file.File} (aka \"{file.Filename}\")");
using var fileWrite = File.OpenWrite(Path.Join(basePath, "assets", file.File));
await (await stream).CopyToAsync(fileWrite);
})
)
);
Console.Out.WriteLine($"All files done");
}
var chapterAnnounces = metadata.Chapters.Select(c => c.Announce ?? c.Start);
accepted.Where(p => chapterAnnounces.Contains(p.Id)).ToList().ForEach(p => {
p.IsChapterAnnounce = true;
p.Chapter = metadata.Chapters.Single(c => (c.Announce ?? c.Start) == p.Id);
});
var razorEngine = new RazorEngine();
var templateFile = "page_template.cshtml";
var baseUrl = "";
var template = razorEngine.Compile<HtmlSafeTemplate>(File.ReadAllText("page_template.cshtml"));
Console.WriteLine($"Using \"{templateFile}\" with base URL {baseUrl}");
string result = template.Run(instance =>
{
instance.Model = new AnonymousTypeWrapper(new
{
Metadata = metadata,
Posts = accepted,
Now = @DateTime.UtcNow,
//BaseUrl = "assets"
BaseUrl = $"/static/{args[0]}"
});
});
Console.WriteLine($"Template output {result.Length} bytes");
File.WriteAllText(Path.Join(basePath, "output.html"), result);
}
public static List<ThreadPost> ProcessLoop(IEnumerable<ThreadPost> allPosts, IEnumerable<string> authors)
{
/*
Add to processing queue:
q/queue #uid
q/queue @name
a/accept #uid
a/accept @name
a/accept #uid >### - accept with post count more than #
i/inspect #uid
i/inspect 12345
i/inspect @name
Drop _only from queue_
d/del/delete/drop #uid
d/del/delete/drop name
Remove from accepted
r/rm/remove #uid
r/rm/remove name
Status/list:
s/l
l uid(s)
l user(s)/name(s)
l accepted
q/quit
h/help this
*/
var accepted = new List<ThreadPost>();
var processing = allPosts.Where(p => authors.Contains(p.Author)).ToHashSet();
Console.WriteLine($"\nProcesing {processing.Count} posts");
{
var grouped = processing.GroupBy(p => new { p.Uid, p.Author }).OrderBy(g => g.Count());
foreach (var item in grouped)
{
Console.Out.WriteLine($"#{item.Key.Uid} - \"{item.Key.Author}\": {item.Count()} posts");
}
}
var loopedWhenEmpty = false;
while (!loopedWhenEmpty || processing.Any()) {
loopedWhenEmpty = !processing.Any();
if (loopedWhenEmpty) {
loopedWhenEmpty = true;
Console.Out.WriteLine("NOTE: No more entries left in queue. This is the last loop unless you run a <list>, <inspect>, or <add> new entries!");
}
Console.Write("> ");
var loopInput = Console.ReadLine();
if (loopInput is null)
break;
loopInput = loopInput.Trim();
if (loopInput == "quit") {
break;
}
var noParam = !loopInput.Contains(' ', StringComparison.InvariantCulture);
var paramType = ParamType.Invalid;
var paramError = ParamError.NoError;
if (loopInput.StartsWith("i ") || loopInput.StartsWith("inspect ")) {
loopedWhenEmpty = false;
var affected = HandleParams(processing, loopInput, out paramType, out paramError);
if (CheckError(paramType, paramError))
continue;
Console.Out.WriteLine($"{affected.Count} items");
foreach (var item in affected) {
var fileDesc = item.File is not null ? $" - File {item.Filename}" : "";
Console.Out.WriteLine($"{item.Id} - {item.Date: s} - {item.Author} - #{item.Uid}{fileDesc}\n\n{item.RawHtml}\n---");
}
continue;
}
if (loopInput.StartsWith("a ") || loopInput.StartsWith("add ") || loopInput.StartsWith("accept ")) {
var affected = new List<ThreadPost>();
if (loopInput == "a all" || loopInput == "add all" || loopInput == "accept all")
affected = processing.ToList();
else
affected = HandleParams(processing, loopInput, out paramType, out paramError);
if (CheckError(paramType, paramError))
continue;
Console.Out.WriteLine($"{affected.Count} items");
processing.RemoveWhere(i => affected.Contains(i));
accepted.AddRange(affected);
continue;
}
if (loopInput.StartsWith("d ") || loopInput.StartsWith("del ") || loopInput.StartsWith("delete ") || loopInput.StartsWith("drop ")) {
var affected = HandleParams(processing, loopInput, out paramType, out paramError);
if (CheckError(paramType, paramError))
continue;
Console.Out.WriteLine($"{affected.Count} items");
processing.RemoveWhere(i => affected.Contains(i));
continue;
}
if (loopInput.StartsWith("q ") || loopInput.StartsWith("queue ")) {
loopedWhenEmpty = false;
var affected = new List<ThreadPost>();
if (loopInput == "q #" || loopInput == "queue #")
affected = allPosts.Where(p => processing.Any(pr => pr.Uid == p.Uid)).ToList();
else
affected = HandleParams(allPosts, loopInput, out paramType, out paramError);
if (CheckError(paramType, paramError))
continue;
Console.Out.WriteLine($"{affected.Count} items");
processing.UnionWith(affected);
continue;
}
if (noParam && (loopInput.StartsWith("l") || loopInput.StartsWith("list"))) {
var grouped = processing.GroupBy(p => new { p.Uid, p.Author }).OrderBy(g => g.Count());
foreach (var item in grouped)
{
Console.Out.WriteLine($"#{item.Key.Uid} - \"{item.Key.Author}\": {item.Count()} posts");
}
continue;
}
if (loopInput.StartsWith("l ") || loopInput.StartsWith("list ")) {
var split = loopInput.Split(" ");
switch (split[1])
{
case "uid":
case "uids": {
var grouped = allPosts.GroupBy(p => new { p.Uid }).OrderBy(g => g.Count());
foreach (var item in grouped)
{
Console.Out.WriteLine($"#{item.Key.Uid}: {item.Count()} posts {item.Where(i => i.File is not null).Count()} files");
}
break;
}
case "user":
case "users":
case "author":
case "authors":
case "name":
case "names":
case "username":
case "usernames": {
var grouped = allPosts.GroupBy(p => new { p.Author }).OrderBy(g => g.Count());
foreach (var item in grouped)
{
Console.Out.WriteLine($"\"{item.Key.Author}\": {item.Count()} posts with {item.Where(i => i.File is not null).Count()} files");
}
break;
}
case "a":
case "accepted": {
var grouped = accepted.GroupBy(p => new { p.Author, p.Uid }).OrderBy(g => g.Count());
foreach (var item in grouped)
{
Console.Out.WriteLine($"\"{item.Key.Author}\" - #{item.Key.Uid}: {item.Count()} posts with {item.Where(i => i.File is not null).Count()} files");
}
break;
}
default:
break;
}
continue;
}
}
return accepted;
}
public static bool CheckError(ParamType paramType, ParamError paramError)
{
if (paramError == ParamError.NoError)
return false;
Console.Out.WriteLine($"Error {paramError} for param {paramType}");
return true;
}
public static List<ThreadPost> HandleParams(IEnumerable<ThreadPost> targetList, string input, out ParamType paramType, out ParamError paramError, int skip = 1)
{
var output = new List<ThreadPost>();
var split = input.Split(" ").Skip(skip);
paramType = ParamType.Invalid;
paramError = ParamError.Invalid;
foreach (var param in split)
{
if (param.StartsWith("#")) {
paramType = ParamType.UniqueId;
paramError = ParamError.NoError;
output.AddRange(targetList.Where(p => p.Uid == param[1..]));
if (!output.Any())
paramError = ParamError.NotFound;
} else if (param.StartsWith("@")) {
paramType = ParamType.Username;
paramError = ParamError.NoError;
var name = param[1..];
output.AddRange(targetList.Where(p => p.Author == name));
if (!output.Any())
paramError = ParamError.NotFound;
} else if (int.TryParse(param, out var postId)) {
paramType = ParamType.PostId;
paramError = ParamError.NoError;
output.AddRange(targetList.Where(p => p.Id == postId));
if (!output.Any())
paramError = ParamError.NotFound;
}
if (paramError != ParamError.NoError) {
return null!;
}
}
return output.OrderBy(i => i.Id).ToList();
}
}
}

65
kusaba.js Normal file
View File

@ -0,0 +1,65 @@
await (async () => {
delete Array.prototype.toJSON;
const processReply = (elem) => {
const id = +elem.querySelector(":scope > .postwidth > a[name]:not([name=s])").getAttribute("name");
const title = elem.querySelector(":scope > .postwidth .filetitle")?.innerText.trim();
const author = elem.querySelector(":scope > .postwidth .postername").innerText.trim();
const uid = elem.querySelector(":scope > .postwidth .uid").innerText.replace("ID: ", "");
const file = elem.querySelector(":scope > .postwidth > .filesize > a")?.href ?? undefined;
const postertrip = elem.querySelector(":scope > .postwidth .postertrip")?.innerText.trim();
const rawHtml = elem.querySelector(":scope > blockquote").innerHTML
.replace(`<div style="display:inline-block; width:400px;"></div><br>`,"")
.trim();
const date = [...elem.querySelector(":scope > .postwidth > label").childNodes]
.pop().data.trim()
.replace(
/(\d{4,4})\/(\d\d)\/(\d\d)\(\w+\)(\d\d):(\d\d)/,
"$1-$2-$3T$4:$5:00Z"
)
.replace(
/href=\\"\/kusaba\/questarch\/res\/\d+.html#\d+\\" onclick=\\"return highlight\('\d+', true\);\\"/,
""
);
const filenameRaw = elem.querySelector(":scope > .postwidth > .filesize")?.innerText;
const filename = filenameRaw?.includes("File ") ?
filenameRaw.match(/File \d+\.[^ ]+ - \([\d\.KMG]+B , \d+x\d+ , (.*) \)/)[1]
?? undefined : undefined;
const ret = {
id,
author,
uid,
rawHtml,
date
}
if (file) ret.file = file;
if (file) ret.filename = filename;
if (postertrip) ret.tripcode = postertrip;
if (title) ret.title = title;
return ret;
}
const replies = [...document.getElementsByClassName("reply")];
replies.unshift(document.getElementById("delform"))
const processed = replies.map(elem => processReply(elem));
const blob = new Blob(
[JSON.stringify(processed, null, 4)],
{type : 'application/json'}
)
const a = document.createElement("a");
const url = URL.createObjectURL(blob);
a.href = url;
a.download = `thread_${processed[0].id}.json`;
document.body.appendChild(a);
a.click();
URL.revokeObjectURL(url);
a.remove();
return ;
})();

75
page_template.cshtml Normal file
View File

@ -0,0 +1,75 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<meta name="color-scheme" content="dark light">
@{
var title = $"\"{Model.Metadata.Name}\" by {Model.Metadata.Author}";
var description = $"Quest single-page archive. Generated {Model.Now} (UTC), {Model.Posts.Count} posts, {Model.Metadata.Chapters.Count} chapters";
// A hack, tbh, should be something better instead..
var preview = $"https://media.lunar.exchange{Model.BaseUrl}/{Model.Metadata.SocialPreview}";
}
<title>@title</title>
<link rel="stylesheet" href="main.css">
<meta name="author" content="@Model.Metadata.Author">
<meta name="description" content="@description">
<meta name="twitter:card" content="summary_large_image"/>
<meta name="twitter:title" content="@title"/>
<meta name="twitter:description" content="@description"/>
<meta name="twitter:image" content="@preview"/>
@if (Model.Metadata.AuthorTwitter is not null) {
<meta name="twitter:creator" content="@("@")@Model.Metadata.AuthorTwitter">
}
<meta name="twitter:site" content="@@SaphireLattice">
<meta property="og:type" content="website"/>
<meta property="og:title" content="@title"/>
<meta property="og:description" content="@description"/>
<meta property="og:determiner" content="the"/>
<meta property="og:locale" content="en_US"/>
<meta property="og:image" content="@preview"/>
<!-- It's just Plausible Analytics - based on ip/etc and daily seed, no incoming requests log. Please email me if you think I am in violation of GDPR/etc -->
<script async defer data-domain="media.lunar.exchange" src="/js/aluminum.js"></script>
</head>
<body>
<header>
<h1><span class="quest-title">@Model.Metadata.Name</span> by <a href="@Model.Metadata.AuthorPage" class="quest-author">@Model.Metadata.Author</a></h1>
<p>@description</p>
<p>Posts from @Model.Posts[0].Date - @Model.Posts[Model.Posts.Count -1].Date</p>
</header>
<main>
@foreach (var item in Model.Posts)
{
@if (item.IsChapterAnnounce) {
<h2 id="chapter-@item.Chapter.Id" class="chapter-announce">
<a class="chapter-anchor" href="#chapter-@item.Chapter.Id">#</a> <span class="chapter-name">@item.Chapter.Name</span> - <span class="chapter-subtitle">@item.Chapter.Subtitle</span>
</h2>
}
<article id="post-@item.Id" class="post@(item.File is not null ? " image-post" : "")">
@if (item.Title is not null) {
<h2 class="post-self-title">@item.Title</h2>
}
<h3 class="post-header"><a class="post-anchor" href="#post-@item.Id"><span class="post-anchor-mark">#</span>@item.Id</a> <span class="author">@item.Author</span> <time>@item.Date</time></h3>
<div class="post-content">
@if (item.File is not null) {
<figure class="post-image">
<img src="@Model.BaseUrl/@item.File" alt="@item.Filename">
</figure>
}
@if (item.RawHtml.Trim().Length > 0) {
<blockquote>@Raw(item.RawHtml)</blockquote>
}
</div>
</article>
}
</main>
<footer>
<p>Ⓒ @Model.Metadata.Author @Model.Posts[0].Date.Year - @Model.Posts[Model.Posts.Count -1].Date.Year, page generated with quest_reader by <a href="https://saphi.re">Saphire</a></p>
</footer>
</body>
</html>

17
quest_reader.csproj Normal file
View File

@ -0,0 +1,17 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net6.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<PropertyGroup>
<GenerateDocumentationFile>true</GenerateDocumentationFile>
<NoWarn>$(NoWarn);1591;8618</NoWarn>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="HtmlAgilityPack" Version="1.11.42" />
<PackageReference Include="RazorEngineCore" Version="2022.1.2" />
</ItemGroup>
</Project>

15
readme.md Normal file
View File

@ -0,0 +1,15 @@
A tool to download and repackage a quest from a quest site, currently supposrts only a Kusaba board (in future: SV and SB)
Possibile stuff to do:
- Multiple post "levels" and actually showing the suggestions
- Story
- Author-Supplementary
- User-supplementary
- Suggestions
- Spam/Junk
- Offline mode!
- Single page/archive download mode?
- Voting for suggestions?
- POSTING NEW ONES???
https://questden.org/kusaba/questarch/src/