-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathSyncService.cs
More file actions
480 lines (425 loc) · 19 KB
/
Copy pathSyncService.cs
File metadata and controls
480 lines (425 loc) · 19 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using PatchNotes.Data;
using PatchNotes.Sync.Core.GitHub;
namespace PatchNotes.Sync.Core;
/// <summary>
/// Service for syncing package releases from GitHub.
/// Designed to be reusable in both console app and Azure Functions.
/// </summary>
public class SyncService
{
private readonly PatchNotesDbContext _db;
private readonly IGitHubClient _github;
private readonly ILogger<SyncService> _logger;
private readonly ChangelogResolver? _changelogResolver;
public SyncService(
PatchNotesDbContext db,
IGitHubClient github,
ILogger<SyncService> logger,
ChangelogResolver? changelogResolver = null)
{
_db = db;
_github = github;
_logger = logger;
_changelogResolver = changelogResolver;
}
/// <summary>
/// Syncs all tracked packages, fetching new releases from GitHub.
/// </summary>
/// <param name="includeExistingWithoutSummary">If true, includes existing releases missing summaries in results.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Result indicating success/failure and statistics.</returns>
public async Task<SyncResult> SyncAllAsync(
bool includeExistingWithoutSummary = false,
CancellationToken cancellationToken = default)
{
var result = new SyncResult();
var packages = await _db.Packages.Where(p => !p.IsSyncDisabled).ToListAsync(cancellationToken);
_logger.LogInformation("Starting sync for {Count} packages", packages.Count);
foreach (var package in packages)
{
if (cancellationToken.IsCancellationRequested)
{
_logger.LogWarning("Sync cancelled");
break;
}
try
{
var packageResult = await SyncPackageAsync(package, includeExistingWithoutSummary, cancellationToken);
result.PackagesSynced++;
result.ReleasesAdded += packageResult.ReleasesAdded;
result.ReleasesNeedingSummary.AddRange(packageResult.ReleasesNeedingSummary);
if (packageResult.ReleasesAdded > 0)
{
_logger.LogInformation(
"Synced {Package}: {Count} new releases, {NeedSummary} need summaries",
package.Name,
packageResult.ReleasesAdded,
packageResult.ReleasesNeedingSummary.Count);
}
else
{
_logger.LogDebug("Synced {Package}: no new releases", package.Name);
}
}
catch (Exception ex)
{
result.Errors.Add(new SyncError(package.Name, ex.Message));
_logger.LogError(ex, "Failed to sync {Package}", package.Name);
// Discard pending changes from the failed package to prevent
// them from being saved by a subsequent package's SaveChanges
foreach (var entry in _db.ChangeTracker.Entries().ToList())
{
if (entry.State == EntityState.Added)
entry.State = EntityState.Detached;
else if (entry.State == EntityState.Modified)
{
entry.CurrentValues.SetValues(entry.OriginalValues);
entry.State = EntityState.Unchanged;
}
}
package.ConsecutiveFailures++;
package.LastFailureAt = DateTimeOffset.UtcNow;
package.LastFailureMessage = ex.Message.Length > 1024 ? ex.Message[..1024] : ex.Message;
await _db.SaveChangesAsync(cancellationToken);
}
}
_logger.LogInformation(
"Sync complete: {Packages} packages, {Releases} new releases, {NeedSummary} need summaries, {Errors} errors",
result.PackagesSynced,
result.ReleasesAdded,
result.ReleasesNeedingSummary.Count,
result.Errors.Count);
return result;
}
/// <summary>
/// Syncs a single package, fetching new releases from GitHub.
/// </summary>
/// <param name="package">The package to sync.</param>
/// <param name="includeExistingWithoutSummary">If true, includes existing releases missing summaries in results.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Result with count of releases added and releases needing summaries.</returns>
public async Task<PackageSyncResult> SyncPackageAsync(
Package package,
bool includeExistingWithoutSummary = false,
CancellationToken cancellationToken = default)
{
if (string.IsNullOrEmpty(package.GithubOwner) || string.IsNullOrEmpty(package.GithubRepo))
{
_logger.LogWarning(
"Skipping {Package}: missing GitHub owner/repo",
package.Name);
return new PackageSyncResult(0);
}
var since = package.LastFetchedAt;
var fetchedAt = DateTimeOffset.UtcNow;
var releasesAdded = 0;
var releasesNeedingSummary = new List<Release>();
// Get existing release tags to avoid duplicates
var existingTags = await _db.Releases
.Where(r => r.PackageId == package.Id)
.Select(r => r.Tag)
.ToHashSetAsync(cancellationToken);
// Batch-collect releases locally before adding to the change tracker.
// This prevents orphaned entities if processing fails mid-loop.
var newReleases = new List<Release>();
await foreach (var ghRelease in _github.GetAllReleasesAsync(
package.GithubOwner,
package.GithubRepo,
cancellationToken))
{
// Skip drafts
if (ghRelease.Draft)
continue;
// Skip releases without a published date
if (!ghRelease.PublishedAt.HasValue)
continue;
// Skip releases that don't match the tag prefix filter
if (!string.IsNullOrEmpty(package.TagPrefix) &&
!ghRelease.TagName.StartsWith(package.TagPrefix, StringComparison.Ordinal))
continue;
// If we have a last fetched date, skip older releases
// GitHub returns releases newest-first, so once we hit an old one, we can stop
if (since.HasValue && ghRelease.PublishedAt.Value <= since.Value)
break;
// Skip if we already have this release
if (existingTags.Contains(ghRelease.TagName))
continue;
var body = ghRelease.Body;
// Follow cross-repo release links (e.g. dotnet/runtime → dotnet/core → dotnet/dotnet)
if (_changelogResolver != null && ChangelogResolver.ExtractGitHubReleaseLink(body) != null)
{
try
{
var followed = await _changelogResolver.FollowReleaseLinksAsync(body, cancellationToken: cancellationToken);
if (followed != null && followed != body)
{
_logger.LogInformation(
"[{Owner}/{Repo}] Followed release links for {Tag}",
package.GithubOwner, package.GithubRepo, ghRelease.TagName);
body = followed;
}
}
catch (Exception ex)
{
_logger.LogWarning(ex,
"[{Owner}/{Repo}] Failed to follow release links for {Tag}, keeping original body",
package.GithubOwner, package.GithubRepo, ghRelease.TagName);
}
}
// Resolve external changelog references
var isRef = ChangelogResolver.IsChangelogReference(body);
_logger.LogInformation(
"[{Owner}/{Repo}] Changelog reference check for {Tag}: isRef={IsRef}, bodyLength={BodyLength}, bodyPreview={BodyPreview}",
package.GithubOwner, package.GithubRepo, ghRelease.TagName, isRef,
body?.Length ?? 0,
body?.Length > 200 ? body[..200] + "…" : body ?? "(null)");
if (_changelogResolver != null && isRef)
{
try
{
var resolved = await _changelogResolver.ResolveAsync(
package.GithubOwner, package.GithubRepo,
ghRelease.TagName, body, cancellationToken);
if (resolved != null)
{
_logger.LogInformation(
"[{Owner}/{Repo}] Resolved changelog reference for {Tag}, resolvedLength={Length}",
package.GithubOwner, package.GithubRepo, ghRelease.TagName, resolved.Length);
body = resolved;
}
else
{
_logger.LogWarning(
"[{Owner}/{Repo}] Changelog resolver returned null for {Tag} — keeping original body",
package.GithubOwner, package.GithubRepo, ghRelease.TagName);
}
}
catch (Exception ex)
{
_logger.LogWarning(ex,
"[{Owner}/{Repo}] Failed to resolve changelog for {Tag}, keeping original body",
package.GithubOwner, package.GithubRepo, ghRelease.TagName);
}
}
var parsed = VersionParser.ParseTagValues(ghRelease.TagName);
var changelogStale = isRef && ChangelogResolver.IsLikelyConventionalCommitsOnly(body);
if (changelogStale)
{
_logger.LogInformation(
"[{Owner}/{Repo}] Flagged {Tag} for changelog re-resolution (bodyLength={Length})",
package.GithubOwner, package.GithubRepo, ghRelease.TagName, body?.Length ?? 0);
}
var release = new Release
{
PackageId = package.Id,
Tag = ghRelease.TagName,
Title = ghRelease.Name,
Body = body,
PublishedAt = ghRelease.PublishedAt.Value,
FetchedAt = fetchedAt,
MajorVersion = parsed.MajorVersion,
MinorVersion = parsed.MinorVersion,
PatchVersion = parsed.PatchVersion,
IsPrerelease = parsed.IsPrerelease,
ChangelogStale = changelogStale
};
newReleases.Add(release);
existingTags.Add(ghRelease.TagName);
releasesAdded++;
// New releases always need summaries
releasesNeedingSummary.Add(release);
}
// Add all releases to the change tracker only after the loop
// completes successfully — no orphans if processing threw mid-loop
_db.Releases.AddRange(newReleases);
package.LastFetchedAt = fetchedAt;
package.ConsecutiveFailures = 0;
package.LastFailureMessage = null;
await _db.SaveChangesAsync(cancellationToken);
// Optionally include existing releases that are missing summaries
if (includeExistingWithoutSummary)
{
var existingWithoutSummary = await _db.Releases
.Where(r => r.PackageId == package.Id && (r.SummaryStale))
.Where(r => !releasesNeedingSummary.Select(x => x.Id).Contains(r.Id))
.ToListAsync(cancellationToken);
releasesNeedingSummary.AddRange(existingWithoutSummary);
}
return new PackageSyncResult(releasesAdded, releasesNeedingSummary);
}
/// <summary>
/// Syncs a single repository by owner/repo, creating the package if it doesn't exist.
/// </summary>
/// <param name="owner">GitHub repository owner.</param>
/// <param name="repo">GitHub repository name.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Result with count of releases added and releases needing summaries.</returns>
public async Task<PackageSyncResult> SyncRepoAsync(
string owner,
string repo,
CancellationToken cancellationToken = default)
{
var package = await _db.Packages
.FirstOrDefaultAsync(
p => p.GithubOwner == owner && p.GithubRepo == repo,
cancellationToken);
var isNewPackage = package == null;
if (isNewPackage)
{
package = new Package
{
Name = repo,
Url = $"https://github.com/{owner}/{repo}",
GithubOwner = owner,
GithubRepo = repo,
};
_db.Packages.Add(package);
await _db.SaveChangesAsync(cancellationToken);
_logger.LogInformation("Created package {Owner}/{Repo}", owner, repo);
}
try
{
return await SyncPackageAsync(package!, cancellationToken: cancellationToken);
}
catch when (isNewPackage)
{
_logger.LogWarning("Sync failed for new package {Owner}/{Repo}, removing phantom package", owner, repo);
_db.Packages.Remove(package!);
await _db.SaveChangesAsync(cancellationToken);
throw;
}
}
/// <summary>
/// Gets all releases that need summary generation across all packages.
/// </summary>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>List of releases needing summaries.</returns>
public async Task<List<Release>> GetReleasesNeedingSummaryAsync(CancellationToken cancellationToken = default)
{
return await _db.Releases
.Include(r => r.Package)
.Where(r => r.SummaryStale)
.OrderByDescending(r => r.PublishedAt)
.ToListAsync(cancellationToken);
}
/// <summary>
/// Gets releases needing summary generation for a specific package.
/// </summary>
/// <param name="packageId">The package ID.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>List of releases needing summaries.</returns>
public async Task<List<Release>> GetReleasesNeedingSummaryAsync(
string packageId,
CancellationToken cancellationToken = default)
{
return await _db.Releases
.Where(r => r.PackageId == packageId && (r.SummaryStale))
.OrderByDescending(r => r.PublishedAt)
.ToListAsync(cancellationToken);
}
/// <summary>
/// Re-attempts changelog resolution for releases flagged as ChangelogStale
/// that were published within the last 12 hours. Clears the flag on expired releases.
/// </summary>
public async Task<int> ReResolveStaleChangelogsAsync(CancellationToken cancellationToken = default)
{
if (_changelogResolver == null)
return 0;
var cutoff = DateTimeOffset.UtcNow.AddHours(-12);
var staleReleases = await _db.Releases
.Include(r => r.Package)
.Where(r => r.ChangelogStale)
.ToListAsync(cancellationToken);
if (staleReleases.Count == 0)
return 0;
var updated = 0;
var expired = 0;
foreach (var release in staleReleases)
{
// Expire releases older than 12 hours
if (release.PublishedAt <= cutoff)
{
release.ChangelogStale = false;
expired++;
_logger.LogDebug("Expiring stale changelog flag for {Tag}", release.Tag);
continue;
}
var package = release.Package;
try
{
var resolved = await _changelogResolver.ResolveAsync(
package.GithubOwner, package.GithubRepo,
release.Tag, release.Body, cancellationToken);
if (resolved != null
&& (resolved.Length > (release.Body?.Length ?? 0)
|| !ChangelogResolver.IsLikelyConventionalCommitsOnly(resolved)))
{
_logger.LogInformation(
"[{Owner}/{Repo}] Re-resolved changelog for {Tag}: {OldLen} -> {NewLen} chars",
package.GithubOwner, package.GithubRepo, release.Tag,
release.Body?.Length ?? 0, resolved.Length);
release.Body = resolved;
release.ChangelogStale = false;
release.SummaryStale = true;
updated++;
}
else
{
_logger.LogDebug(
"[{Owner}/{Repo}] Re-resolution for {Tag} did not improve changelog, will retry",
package.GithubOwner, package.GithubRepo, release.Tag);
}
}
catch (Exception ex)
{
_logger.LogWarning(ex,
"[{Owner}/{Repo}] Failed to re-resolve changelog for {Tag}",
package.GithubOwner, package.GithubRepo, release.Tag);
}
}
if (updated > 0 || expired > 0)
{
await _db.SaveChangesAsync(cancellationToken);
_logger.LogInformation(
"Stale changelog re-resolution: {Updated} updated, {Expired} expired",
updated, expired);
}
return updated;
}
/// <summary>
/// Backfills denormalized version fields for all existing releases.
/// Parses the Tag and updates MajorVersion, MinorVersion, PatchVersion, and IsPrerelease.
/// Safe to call multiple times (idempotent).
/// </summary>
/// <returns>Number of releases updated.</returns>
public async Task<int> BackfillVersionFieldsAsync(CancellationToken cancellationToken = default)
{
var releases = await _db.Releases
.Where(r => r.MajorVersion == 0 && r.MinorVersion == 0 && r.PatchVersion == 0)
.ToListAsync(cancellationToken);
var updated = 0;
foreach (var release in releases)
{
var parsed = VersionParser.ParseTagValues(release.Tag);
if (release.MajorVersion != parsed.MajorVersion
|| release.MinorVersion != parsed.MinorVersion
|| release.PatchVersion != parsed.PatchVersion
|| release.IsPrerelease != parsed.IsPrerelease)
{
release.MajorVersion = parsed.MajorVersion;
release.MinorVersion = parsed.MinorVersion;
release.PatchVersion = parsed.PatchVersion;
release.IsPrerelease = parsed.IsPrerelease;
updated++;
}
}
if (updated > 0)
{
await _db.SaveChangesAsync(cancellationToken);
_logger.LogInformation("Backfilled version fields for {Count} releases", updated);
}
return updated;
}
}