sjones/PocketRSSSync
Archived
1
0
Fork 0
Code Activity

parallelize feed parsing and lessen pocket query

Browse Source
This commit is contained in:
Spencer Jones
2022-05-30 11:32:35 -07:00
parent 6a97816e33
commit e717fb7427
3 changed files with 51 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
PocketRSSSync/Models/PocketItem.cs
View File

@@ -24,7 +24,7 @@
{ {
consumer_key = auth.ConsumerKey, consumer_key = auth.ConsumerKey,
access_token = auth.AccessToken, access_token = auth.AccessToken,
detailType = "complete", detailType = "simple",
state = "all" state = "all"
}); });
request.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json"); request.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
@@ -41,6 +41,8 @@
var items = new List<PocketItem>(); var items = new List<PocketItem>();
foreach (var listItem in dynConfig.list) foreach (var listItem in dynConfig.list)
{
if (listItem.Value.status != "2")
{ {
items.Add(new PocketItem() items.Add(new PocketItem()
{ {
@@ -48,6 +50,7 @@
Url = listItem.Value.given_url Url = listItem.Value.given_url
}); });
} }
}
return items; return items;
} }

2
PocketRSSSync/PocketRSSSync.csproj
View File

@@ -10,7 +10,7 @@
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="Microsoft.Extensions.Hosting" Version="6.0.0" /> <PackageReference Include="Microsoft.Extensions.Hosting" Version="6.0.1" />
<PackageReference Include="Microsoft.Extensions.Hosting.WindowsServices" Version="6.0.0" /> <PackageReference Include="Microsoft.Extensions.Hosting.WindowsServices" Version="6.0.0" />
<PackageReference Include="Microsoft.Extensions.Http" Version="6.0.0" /> <PackageReference Include="Microsoft.Extensions.Http" Version="6.0.0" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.1" /> <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />

27
PocketRSSSync/Worker.cs
View File

@@ -5,7 +5,6 @@ namespace PocketRSSSync
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using PocketRSSSync.Models; using PocketRSSSync.Models;
using RssFeedParser; using RssFeedParser;
using RssFeedParser.Models;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
@@ -43,9 +42,21 @@ namespace PocketRSSSync
var feedUris = config.GetSection("Feeds").Get<List<string>>(); var feedUris = config.GetSection("Feeds").Get<List<string>>();
var taskList = new List<Task<RssFeed>>(); var taskList = new List<Task>();
foreach (var feedUri in feedUris) foreach (var feedUri in feedUris)
{
taskList.Add(ProcessFeed(feedUri, currentItems));
}
Task.WaitAll(taskList.ToArray());
logger.LogInformation("{count} total items added to Pocket since program starting. Worker running at: {time}", count, DateTimeOffset.Now);
await Task.Delay(new TimeSpan(2, 0, 0), stoppingToken);
}
}
private async Task ProcessFeed(string feedUri, List<PocketItem> currentItems)
{ {
try try
{ {
@@ -78,17 +89,7 @@ namespace PocketRSSSync
catch (HttpRequestException ex) catch (HttpRequestException ex)
{ {
if (ex.StatusCode != System.Net.HttpStatusCode.NotFound) logger.LogError($"{feedUri} returned {ex.StatusCode}");
{
throw;
}
logger.LogError($"{feedUri} returned {ex.StatusCode.ToString()}");
}
}
logger.LogInformation("{count} total items added to Pocket since program starting. Worker running at: {time}", count, DateTimeOffset.Now);
await Task.Delay(new TimeSpan(2, 0, 0), stoppingToken);
} }
} }
} }