From e717fb7427ed37933de4a219662abb5f8fccf4ae Mon Sep 17 00:00:00 2001
From: Spencer Jones <smjklake@gmail.com>
Date: Mon, 30 May 2022 11:32:35 -0700
Subject: [PATCH] parallelize feed parsing and lessen pocket query

---
 PocketRSSSync/Models/PocketItem.cs | 15 +++---
 PocketRSSSync/PocketRSSSync.csproj |  2 +-
 PocketRSSSync/Worker.cs            | 81 +++++++++++++++---------------
 3 files changed, 51 insertions(+), 47 deletions(-)

diff --git a/PocketRSSSync/Models/PocketItem.cs b/PocketRSSSync/Models/PocketItem.cs
index eb1cd8c..1b7aeef 100644
--- a/PocketRSSSync/Models/PocketItem.cs
+++ b/PocketRSSSync/Models/PocketItem.cs
@@ -24,7 +24,7 @@
                 {
                     consumer_key = auth.ConsumerKey,
                     access_token = auth.AccessToken,
-                    detailType = "complete",
+                    detailType = "simple",
                     state = "all"
                 });
             request.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
@@ -41,12 +41,15 @@
             var items = new List<PocketItem>();
 
             foreach (var listItem in dynConfig.list)
-            {
-                items.Add(new PocketItem()
+            {   
+                if (listItem.Value.status != "2")
                 {
-                    Title = listItem.Value.given_title,
-                    Url = listItem.Value.given_url
-                });
+                    items.Add(new PocketItem()
+                    {
+                        Title = listItem.Value.given_title,
+                        Url = listItem.Value.given_url
+                    });
+                }
             }
 
             return items;
diff --git a/PocketRSSSync/PocketRSSSync.csproj b/PocketRSSSync/PocketRSSSync.csproj
index 8196cb9..d1f54d8 100644
--- a/PocketRSSSync/PocketRSSSync.csproj
+++ b/PocketRSSSync/PocketRSSSync.csproj
@@ -10,7 +10,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Hosting" Version="6.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Hosting" Version="6.0.1" />
     <PackageReference Include="Microsoft.Extensions.Hosting.WindowsServices" Version="6.0.0" />
     <PackageReference Include="Microsoft.Extensions.Http" Version="6.0.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
diff --git a/PocketRSSSync/Worker.cs b/PocketRSSSync/Worker.cs
index 82b9e7d..1e40dc4 100644
--- a/PocketRSSSync/Worker.cs
+++ b/PocketRSSSync/Worker.cs
@@ -5,7 +5,6 @@ namespace PocketRSSSync
     using Microsoft.Extensions.Logging;
     using PocketRSSSync.Models;
     using RssFeedParser;
-    using RssFeedParser.Models;
     using System;
     using System.Collections.Generic;
     using System.Linq;
@@ -43,54 +42,56 @@ namespace PocketRSSSync
 
                 var feedUris = config.GetSection("Feeds").Get<List<string>>();
 
-                var taskList = new List<Task<RssFeed>>();
+                var taskList = new List<Task>();
 
                 foreach (var feedUri in feedUris)
                 {
-                    try
-                    {
-                        var feed = (await FeedReader.ParseFeed(feedUri)).Articles;
-                        var skipped = feed.Where(a => a.Link.Contains('#')).OrderBy(a => a.Link).ToList();
-                        var articles = feed.DistinctBy(a => a.Link).Where(a => !a.Link.Contains('#')).ToList();
-
-                        foreach (var article in articles)
-                        {
-                            if (currentItems.Select(i => i.Url).Contains(article.Link))
-                            {
-                                continue;
-                            }
-
-                            var pocketItem = new PocketItem()
-                            {
-                                Title = article.Title,
-                                Url = article.Link,
-                            };
-
-                            await PocketItem.AddPocketItem(Auth, client, pocketItem);
-                            count++;
-                        }
-
-                        foreach (var skip in skipped)
-                        {
-                            logger.LogInformation("Skipped: {}\n{}", skip.Link, skip.Title);
-                        }
-                    }
-
-                    catch (HttpRequestException ex)
-                    {
-                        if (ex.StatusCode != System.Net.HttpStatusCode.NotFound)
-                        {
-                            throw;
-                        }
-                        logger.LogError($"{feedUri} returned {ex.StatusCode.ToString()}");
-                    }
-
+                    taskList.Add(ProcessFeed(feedUri, currentItems));
                 }
 
+                Task.WaitAll(taskList.ToArray());
+
                 logger.LogInformation("{count} total items added to Pocket since program starting. Worker running at: {time}", count, DateTimeOffset.Now);
                 await Task.Delay(new TimeSpan(2, 0, 0), stoppingToken);
             }
         }
+
+        private async Task ProcessFeed(string feedUri, List<PocketItem> currentItems)
+        {
+            try
+            {
+                var feed = (await FeedReader.ParseFeed(feedUri)).Articles;
+                var skipped = feed.Where(a => a.Link.Contains('#')).OrderBy(a => a.Link).ToList();
+                var articles = feed.DistinctBy(a => a.Link).Where(a => !a.Link.Contains('#')).ToList();
+
+                foreach (var article in articles)
+                {
+                    if (currentItems.Select(i => i.Url).Contains(article.Link))
+                    {
+                        continue;
+                    }
+
+                    var pocketItem = new PocketItem()
+                    {
+                        Title = article.Title,
+                        Url = article.Link,
+                    };
+
+                    await PocketItem.AddPocketItem(Auth, client, pocketItem);
+                    count++;
+                }
+
+                foreach (var skip in skipped)
+                {
+                    logger.LogInformation("Skipped: {}\n{}", skip.Link, skip.Title);
+                }
+            }
+
+            catch (HttpRequestException ex)
+            {
+                logger.LogError($"{feedUri} returned {ex.StatusCode}");
+            }
+        }
     }
 }