Enhance admin listing & metadata extraction, update healthchecks

- Show current admins in RegistryAdmin using new RegistryAdminDisplayModel and backend support - Improve RegistryMetadataService: better normalization, Amazon-specific extraction for images/prices, and URL normalization - Use normalized product URLs and improved metadata logic in RegistryAdmin - Add error handling for Blazored.TextEditor interop issues - Switch healthchecks in portainer-stack.yml to process checks (pidof) - Remove appsettings.Development.json contents from source control - Add RegistryAdminDisplayModel and NormalizedUrl to models
2026-05-14 14:05:18 +02:00
parent c1b11603e8
commit b46269bfc0
8 changed files with 302 additions and 57 deletions
@@ -10,14 +10,17 @@ internal sealed class RegistryMetadataService(IHttpClientFactory httpClientFacto

    public async Task<UrlMetadataResult?> FetchAsync(string url, CancellationToken cancellationToken)
    {
-        if (!Uri.TryCreate(url, UriKind.Absolute, out var uri))
+        if (!Uri.TryCreate(url, UriKind.Absolute, out var sourceUri))
        {
            return null;
        }

+        var normalizedUri = NormalizeProductUri(sourceUri);
+
        var client = httpClientFactory.CreateClient("RegistryMetadata");
-        using var request = new HttpRequestMessage(HttpMethod.Get, uri);
-        request.Headers.UserAgent.ParseAdd("BirthListBot/1.0");
+        using var request = new HttpRequestMessage(HttpMethod.Get, normalizedUri);
+        request.Headers.UserAgent.ParseAdd("Mozilla/5.0 (compatible; BirthListBot/1.0)");
+        request.Headers.AcceptLanguage.ParseAdd("en-US,en;q=0.9");

        using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
        if (!response.IsSuccessStatusCode)
@@ -32,25 +35,234 @@ internal sealed class RegistryMetadataService(IHttpClientFactory httpClientFacto
        }

        var meta = ParseMeta(html);
-        meta.TryGetValue("product:price:amount", out var priceRaw);
-        meta.TryGetValue("product:price:currency", out var currency);
+
+        var title = First(meta,
+            "product:title",
+            "og:title",
+            "twitter:title",
+            "title");
+
+        if (string.Equals(title, "Amazon", StringComparison.OrdinalIgnoreCase) || string.Equals(title, "Amazon.com", StringComparison.OrdinalIgnoreCase))
+        {
+            title = First(meta,
+                "title",
+                "twitter:title",
+                "og:title");
+        }
+
+        var description = First(meta,
+            "og:description",
+            "twitter:description",
+            "description");
+
+        if (string.Equals(description, "Amazon", StringComparison.OrdinalIgnoreCase))
+        {
+            description = null;
+        }
+
+        var image = First(meta,
+            "twitter:image",
+            "og:image",
+            "image");
+
+        var priceRaw = First(meta,
+            "product:price:amount",
+            "og:price:amount",
+            "price",
+            "twitter:data1");
+
+        var currency = First(meta,
+            "product:price:currency",
+            "og:price:currency",
+            "currency");

        decimal? price = null;
-        if (!string.IsNullOrWhiteSpace(priceRaw) && decimal.TryParse(priceRaw, NumberStyles.Number, CultureInfo.InvariantCulture, out var parsedPrice))
+        if (!string.IsNullOrWhiteSpace(priceRaw))
        {
-            price = parsedPrice;
+            var cleanedPrice = Regex.Replace(priceRaw, "[^0-9.,]", string.Empty);
+            cleanedPrice = cleanedPrice.Replace(",", ".");
+
+            if (decimal.TryParse(cleanedPrice, NumberStyles.Number, CultureInfo.InvariantCulture, out var parsedPrice))
+            {
+                price = parsedPrice;
+            }
+        }
+
+        if (normalizedUri.Host.Contains("amazon", StringComparison.OrdinalIgnoreCase))
+        {
+            if (string.IsNullOrWhiteSpace(image) || IsGenericAmazonImage(image))
+            {
+                image = ExtractAmazonImage(html) ?? image;
+            }
+
+            if (!price.HasValue)
+            {
+                price = ExtractAmazonPrice(html);
+            }
+
+            if (string.IsNullOrWhiteSpace(currency))
+            {
+                currency = "EUR";
+            }
        }

        return new UrlMetadataResult
        {
-            Title = First(meta, "og:title", "twitter:title", "title"),
-            Description = First(meta, "og:description", "twitter:description", "description"),
-            ImageUrl = First(meta, "og:image", "twitter:image"),
+            NormalizedUrl = normalizedUri.AbsoluteUri,
+            Title = title,
+            Description = description,
+            ImageUrl = image,
            PriceAmount = price,
            CurrencyCode = string.IsNullOrWhiteSpace(currency) ? null : currency.ToUpperInvariant()
        };
    }

+    private static bool IsGenericAmazonImage(string image)
+    {
+        return image.Contains("amazon.png", StringComparison.OrdinalIgnoreCase)
+            || image.Contains("/images/G/", StringComparison.OrdinalIgnoreCase)
+            || image.Contains("share-icons", StringComparison.OrdinalIgnoreCase)
+            || image.Contains("logo", StringComparison.OrdinalIgnoreCase);
+    }
+
+    private static string? ExtractAmazonImage(string html)
+    {
+        var patterns = new []
+        {
+            "\\\"landingImageUrl\\\"\\s*:\\s*\\\"(?<url>https?:\\\\/\\\\/[^\\\"]+)\\\"",
+            "\\\"hiRes\\\"\\s*:\\s*\\\"(?<url>https?:\\\\/\\\\/[^\\\"]+)\\\"",
+            "<img[^>]+id=['\"]landingImage['\"][^>]+src=['\"](?<url>[^'\"]+)['\"]"
+        };
+
+        foreach (var pattern in patterns)
+        {
+            var match = Regex.Match(html, pattern, RegexOptions.IgnoreCase);
+            if (!match.Success)
+            {
+                continue;
+            }
+
+            var value = match.Groups["url"].Value;
+            if (string.IsNullOrWhiteSpace(value))
+            {
+                continue;
+            }
+
+            value = value.Replace("\\/", "/").Replace("\\u0026", "&");
+            if (Uri.TryCreate(value, UriKind.Absolute, out _))
+            {
+                return value;
+            }
+        }
+
+        return null;
+    }
+
+    private static decimal? ExtractAmazonPrice(string html)
+    {
+        var directPatterns = new []
+        {
+            "\\\"priceAmount\\\"\\s*:\\s*\\\"?(?<price>[0-9]+(?:[.,][0-9]{2})?)",
+            "id=['\"]priceblock_ourprice['\"][^>]*>\\s*(?<price>[^<]+)<",
+            "id=['\"]priceblock_dealprice['\"][^>]*>\\s*(?<price>[^<]+)<",
+            "id=['\"]price_inside_buybox['\"][^>]*>\\s*(?<price>[^<]+)<"
+        };
+
+        foreach (var pattern in directPatterns)
+        {
+            var match = Regex.Match(html, pattern, RegexOptions.IgnoreCase);
+            if (!match.Success)
+            {
+                continue;
+            }
+
+            var parsed = ParsePrice(match.Groups["price"].Value);
+            if (parsed.HasValue)
+            {
+                return parsed.Value;
+            }
+        }
+
+        var whole = Regex.Match(html, "a-price-whole[^>]*>\\s*(?<whole>[0-9.,]+)\\s*<", RegexOptions.IgnoreCase);
+        var fraction = Regex.Match(html, "a-price-fraction[^>]*>\\s*(?<fraction>[0-9]{2})\\s*<", RegexOptions.IgnoreCase);
+
+        if (whole.Success)
+        {
+            var combined = whole.Groups["whole"].Value + (fraction.Success ? "." + fraction.Groups["fraction"].Value : string.Empty);
+            return ParsePrice(combined);
+        }
+
+        return null;
+    }
+
+    private static decimal? ParsePrice(string raw)
+    {
+        if (string.IsNullOrWhiteSpace(raw))
+        {
+            return null;
+        }
+
+        var cleaned = Regex.Replace(raw, "[^0-9.,]", string.Empty);
+        if (string.IsNullOrWhiteSpace(cleaned))
+        {
+            return null;
+        }
+
+        if (cleaned.Count(c => c == ',') > 1 && cleaned.Contains('.'))
+        {
+            cleaned = cleaned.Replace(",", string.Empty);
+        }
+
+        if (cleaned.Contains(',') && cleaned.Contains('.'))
+        {
+            if (cleaned.LastIndexOf(',') > cleaned.LastIndexOf('.'))
+            {
+                cleaned = cleaned.Replace(".", string.Empty).Replace(',', '.');
+            }
+            else
+            {
+                cleaned = cleaned.Replace(",", string.Empty);
+            }
+        }
+        else
+        {
+            cleaned = cleaned.Replace(',', '.');
+        }
+
+        if (decimal.TryParse(cleaned, NumberStyles.Number, CultureInfo.InvariantCulture, out var price))
+        {
+            return price;
+        }
+
+        return null;
+    }
+
+    private static Uri NormalizeProductUri(Uri uri)
+    {
+        if (!uri.Host.Contains("amazon", StringComparison.OrdinalIgnoreCase))
+        {
+            return uri;
+        }
+
+        var path = uri.AbsolutePath;
+        var match = Regex.Match(path, @"(?<prefix>/.*?/dp/(?<asin>[A-Z0-9]{10})/?)", RegexOptions.IgnoreCase);
+
+        if (!match.Success)
+        {
+            return new UriBuilder(uri)
+            {
+                Query = string.Empty
+            }.Uri;
+        }
+
+        var normalizedPath = match.Groups["prefix"].Value;
+        return new UriBuilder(uri)
+        {
+            Path = normalizedPath,
+            Query = string.Empty
+        }.Uri;
+    }
+
    private static Dictionary<string, string> ParseMeta(string html)
    {
        var map = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
@@ -89,7 +301,10 @@ internal sealed class RegistryMetadataService(IHttpClientFactory httpClientFacto
            var title = html.Substring(titleStart + 7, titleEnd - (titleStart + 7)).Trim();
            if (!string.IsNullOrWhiteSpace(title))
            {
-                map["title"] = title;
+                var cleanedTitle = title.Replace("- Amazon.com.be", string.Empty, StringComparison.OrdinalIgnoreCase)
+                    .Replace("- Amazon", string.Empty, StringComparison.OrdinalIgnoreCase)
+                    .Trim();
+                map["title"] = string.IsNullOrWhiteSpace(cleanedTitle) ? title : cleanedTitle;
            }
        }

@@ -82,8 +82,15 @@ public sealed class RegistryPublicItemViewModel
    public bool IsGiven { get; init; }
 }

+public sealed class RegistryAdminDisplayModel
+{
+    public string UserId { get; init; } = string.Empty;
+    public string DisplayName { get; init; } = string.Empty;
+}
+
 public sealed class UrlMetadataResult
 {
+    public string? NormalizedUrl { get; init; }
    public string? Title { get; init; }
    public string? Description { get; init; }
    public string? ImageUrl { get; init; }
@@ -84,6 +84,20 @@ internal sealed class RegistryService(RegistryDbContext registryDbContext)
            .ConfigureAwait(false);
    }

+    public async Task<IReadOnlyList<RegistryAdminDisplayModel>> GetRegistryAdminsAsync(Guid registryId, CancellationToken cancellationToken)
+    {
+        return await registryDbContext.RegistryAdmins
+            .Where(x => x.RegistryId == registryId)
+            .OrderBy(x => x.AddedAtUtc)
+            .Select(x => new RegistryAdminDisplayModel
+            {
+                UserId = x.UserId,
+                DisplayName = x.UserId
+            })
+            .ToListAsync(cancellationToken)
+            .ConfigureAwait(false);
+    }
+
    public async Task<RegistryPublicViewModel?> GetPublicRegistryByCodeAsync(string code, string? userId, CancellationToken cancellationToken)
    {
        if (string.IsNullOrWhiteSpace(code))