make the feeds service only update

feeds/handler/crawl.go

@@ -4,8 +4,8 @@ import (
 	"context"
 	"crypto/md5"
 	"fmt"
-
 	"net/url"
+	"sync"
 
 	"github.com/SlyMarbo/rss"
 	log "github.com/micro/micro/v3/service/logger"
@@ -14,6 +14,11 @@ import (
 	posts "github.com/micro/services/posts/proto"
 )
 
+var (
+	rssSync  sync.RWMutex
+	rssFeeds = map[string]*rss.Feed{}
+)
+
 func (e *Feeds) fetchAll() {
 	fs := []*feeds.Feed{}
 	err := e.feeds.Read(e.feedsNameIndex.ToQuery(nil), &fs)
@@ -35,18 +40,40 @@ func (e *Feeds) fetchAll() {
 
 func (e *Feeds) fetch(f *feeds.Feed) error {
 	url := f.Url
-
 	log.Infof("Fetching address %v", url)
-	fd, err := rss.Fetch(url)
-	if err != nil {
-		return fmt.Errorf("Error fetching address %v: %v", url, err)
+
+	// see if there's an existing rss feed
+	rssSync.RLock()
+	fd, ok := rssFeeds[f.Name]
+	rssSync.RUnlock()
+
+	if !ok {
+		// create a new one if it doesn't exist
+		var err error
+		fd, err = rss.Fetch(f.Url)
+		if err != nil {
+			return fmt.Errorf("Error fetching address %v: %v", url, err)
+		}
+		// save the feed
+		rssSync.Lock()
+		rssFeeds[f.Name] = fd
+		rssSync.Unlock()
+	} else {
+		// otherwise update the existing feed
+		fd.Items = []*rss.Item{}
+		fd.Unread = 0
+		if err := fd.Update(); err != nil {
+			return fmt.Errorf("Error updating address %v: %v", url, err)
+		}
 	}
+
 	domain := getDomain(url)
 
+	// range over the feed and save the items
 	for _, item := range fd.Items {
 		id := fmt.Sprintf("%x", md5.Sum([]byte(item.ID)))
 
-		err = e.entries.Create(feeds.Entry{
+		err := e.entries.Create(feeds.Entry{
 			Id:       id,
 			Url:      item.Link,
 			Title:    item.Title,