sjones/PocketRSSSync
Archived
1
0
Fork 0
Code Activity

parallelize feed parsing and lessen pocket query

Browse Source
This commit is contained in:
Spencer Jones
2022-05-30 11:32:35 -07:00
parent 6a97816e33
commit e717fb7427
3 changed files with 51 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
PocketRSSSync/Models/PocketItem.cs
View File

@@ -24,7 +24,7 @@
{
consumer_key = auth.ConsumerKey,
access_token = auth.AccessToken,
detailType = "complete",
detailType = "simple",
state = "all"
});
request.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
@@ -42,11 +42,14 @@
foreach (var listItem in dynConfig.list)
{
items.Add(new PocketItem()
if (listItem.Value.status != "2")
{
Title = listItem.Value.given_title,
Url = listItem.Value.given_url
});
items.Add(new PocketItem()
{
Title = listItem.Value.given_title,
Url = listItem.Value.given_url
});
}
}
return items;

2
PocketRSSSync/PocketRSSSync.csproj
View File

@@ -10,7 +10,7 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Hosting" Version="6.0.0" />
<PackageReference Include="Microsoft.Extensions.Hosting" Version="6.0.1" />
<PackageReference Include="Microsoft.Extensions.Hosting.WindowsServices" Version="6.0.0" />
<PackageReference Include="Microsoft.Extensions.Http" Version="6.0.0" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.1" />

81
PocketRSSSync/Worker.cs
View File

@@ -5,7 +5,6 @@ namespace PocketRSSSync
using Microsoft.Extensions.Logging;
using PocketRSSSync.Models;
using RssFeedParser;
using RssFeedParser.Models;
using System;
using System.Collections.Generic;
using System.Linq;
@@ -43,54 +42,56 @@ namespace PocketRSSSync
var feedUris = config.GetSection("Feeds").Get<List<string>>();
var taskList = new List<Task<RssFeed>>();
var taskList = new List<Task>();
foreach (var feedUri in feedUris)
{
try
{
var feed = (await FeedReader.ParseFeed(feedUri)).Articles;
var skipped = feed.Where(a => a.Link.Contains('#')).OrderBy(a => a.Link).ToList();
var articles = feed.DistinctBy(a => a.Link).Where(a => !a.Link.Contains('#')).ToList();
foreach (var article in articles)
{
if (currentItems.Select(i => i.Url).Contains(article.Link))
{
continue;
}
var pocketItem = new PocketItem()
{
Title = article.Title,
Url = article.Link,
};
await PocketItem.AddPocketItem(Auth, client, pocketItem);
count++;
}
foreach (var skip in skipped)
{
logger.LogInformation("Skipped: {}\n{}", skip.Link, skip.Title);
}
}
catch (HttpRequestException ex)
{
if (ex.StatusCode != System.Net.HttpStatusCode.NotFound)
{
throw;
}
logger.LogError($"{feedUri} returned {ex.StatusCode.ToString()}");
}
taskList.Add(ProcessFeed(feedUri, currentItems));
}
Task.WaitAll(taskList.ToArray());
logger.LogInformation("{count} total items added to Pocket since program starting. Worker running at: {time}", count, DateTimeOffset.Now);
await Task.Delay(new TimeSpan(2, 0, 0), stoppingToken);
}
}
private async Task ProcessFeed(string feedUri, List<PocketItem> currentItems)
{
try
{
var feed = (await FeedReader.ParseFeed(feedUri)).Articles;
var skipped = feed.Where(a => a.Link.Contains('#')).OrderBy(a => a.Link).ToList();
var articles = feed.DistinctBy(a => a.Link).Where(a => !a.Link.Contains('#')).ToList();
foreach (var article in articles)
{
if (currentItems.Select(i => i.Url).Contains(article.Link))
{
continue;
}
var pocketItem = new PocketItem()
{
Title = article.Title,
Url = article.Link,
};
await PocketItem.AddPocketItem(Auth, client, pocketItem);
count++;
}
foreach (var skip in skipped)
{
logger.LogInformation("Skipped: {}\n{}", skip.Link, skip.Title);
}
}
catch (HttpRequestException ex)
{
logger.LogError($"{feedUri} returned {ex.StatusCode}");
}
}
}
}