parallelize feed parsing and lessen pocket query

PocketRSSSync/Models/PocketItem.cs

@@ -24,7 +24,7 @@
                 {
                     consumer_key = auth.ConsumerKey,
                     access_token = auth.AccessToken,
-                    detailType = "complete",
+                    detailType = "simple",
                     state = "all"
                 });
             request.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
@@ -42,11 +42,14 @@
 
             foreach (var listItem in dynConfig.list)
             {   
-                items.Add(new PocketItem()
+                if (listItem.Value.status != "2")
                 {
-                    Title = listItem.Value.given_title,
+                    items.Add(new PocketItem()
-                    Url = listItem.Value.given_url
+                    {
-                });
+                        Title = listItem.Value.given_title,
+                        Url = listItem.Value.given_url
+                    });
+                }
             }
 
             return items;

PocketRSSSync/PocketRSSSync.csproj

@@ -10,7 +10,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Hosting" Version="6.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Hosting" Version="6.0.1" />
     <PackageReference Include="Microsoft.Extensions.Hosting.WindowsServices" Version="6.0.0" />
     <PackageReference Include="Microsoft.Extensions.Http" Version="6.0.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />

PocketRSSSync/Worker.cs

@@ -5,7 +5,6 @@ namespace PocketRSSSync
     using Microsoft.Extensions.Logging;
     using PocketRSSSync.Models;
     using RssFeedParser;
-    using RssFeedParser.Models;
     using System;
     using System.Collections.Generic;
     using System.Linq;
@@ -43,54 +42,56 @@ namespace PocketRSSSync
 
                 var feedUris = config.GetSection("Feeds").Get<List<string>>();
 
-                var taskList = new List<Task<RssFeed>>();
+                var taskList = new List<Task>();
 
                 foreach (var feedUri in feedUris)
                 {
-                    try
+                    taskList.Add(ProcessFeed(feedUri, currentItems));
-                    {
-                        var feed = (await FeedReader.ParseFeed(feedUri)).Articles;
-                        var skipped = feed.Where(a => a.Link.Contains('#')).OrderBy(a => a.Link).ToList();
-                        var articles = feed.DistinctBy(a => a.Link).Where(a => !a.Link.Contains('#')).ToList();
-
-                        foreach (var article in articles)
-                        {
-                            if (currentItems.Select(i => i.Url).Contains(article.Link))
-                            {
-                                continue;
-                            }
-
-                            var pocketItem = new PocketItem()
-                            {
-                                Title = article.Title,
-                                Url = article.Link,
-                            };
-
-                            await PocketItem.AddPocketItem(Auth, client, pocketItem);
-                            count++;
-                        }
-
-                        foreach (var skip in skipped)
-                        {
-                            logger.LogInformation("Skipped: {}\n{}", skip.Link, skip.Title);
-                        }
-                    }
-
-                    catch (HttpRequestException ex)
-                    {
-                        if (ex.StatusCode != System.Net.HttpStatusCode.NotFound)
-                        {
-                            throw;
-                        }
-                        logger.LogError($"{feedUri} returned {ex.StatusCode.ToString()}");
-                    }
-
                 }
 
+                Task.WaitAll(taskList.ToArray());
+
                 logger.LogInformation("{count} total items added to Pocket since program starting. Worker running at: {time}", count, DateTimeOffset.Now);
                 await Task.Delay(new TimeSpan(2, 0, 0), stoppingToken);
             }
         }
+
+        private async Task ProcessFeed(string feedUri, List<PocketItem> currentItems)
+        {
+            try
+            {
+                var feed = (await FeedReader.ParseFeed(feedUri)).Articles;
+                var skipped = feed.Where(a => a.Link.Contains('#')).OrderBy(a => a.Link).ToList();
+                var articles = feed.DistinctBy(a => a.Link).Where(a => !a.Link.Contains('#')).ToList();
+
+                foreach (var article in articles)
+                {
+                    if (currentItems.Select(i => i.Url).Contains(article.Link))
+                    {
+                        continue;
+                    }
+
+                    var pocketItem = new PocketItem()
+                    {
+                        Title = article.Title,
+                        Url = article.Link,
+                    };
+
+                    await PocketItem.AddPocketItem(Auth, client, pocketItem);
+                    count++;
+                }
+
+                foreach (var skip in skipped)
+                {
+                    logger.LogInformation("Skipped: {}\n{}", skip.Link, skip.Title);
+                }
+            }
+
+            catch (HttpRequestException ex)
+            {
+                logger.LogError($"{feedUri} returned {ex.StatusCode}");
+            }
+        }
     }
 }