diff --git a/feeds/handler/crawl.go b/feeds/handler/crawl.go index 56ee741..c56e5e0 100644 --- a/feeds/handler/crawl.go +++ b/feeds/handler/crawl.go @@ -4,8 +4,8 @@ import ( "context" "crypto/md5" "fmt" - "net/url" + "sync" "github.com/SlyMarbo/rss" log "github.com/micro/micro/v3/service/logger" @@ -14,6 +14,11 @@ import ( posts "github.com/micro/services/posts/proto" ) +var ( + rssSync sync.RWMutex + rssFeeds = map[string]*rss.Feed{} +) + func (e *Feeds) fetchAll() { fs := []*feeds.Feed{} err := e.feeds.Read(e.feedsNameIndex.ToQuery(nil), &fs) @@ -35,18 +40,40 @@ func (e *Feeds) fetchAll() { func (e *Feeds) fetch(f *feeds.Feed) error { url := f.Url - log.Infof("Fetching address %v", url) - fd, err := rss.Fetch(url) - if err != nil { - return fmt.Errorf("Error fetching address %v: %v", url, err) + + // see if there's an existing rss feed + rssSync.RLock() + fd, ok := rssFeeds[f.Name] + rssSync.RUnlock() + + if !ok { + // create a new one if it doesn't exist + var err error + fd, err = rss.Fetch(f.Url) + if err != nil { + return fmt.Errorf("Error fetching address %v: %v", url, err) + } + // save the feed + rssSync.Lock() + rssFeeds[f.Name] = fd + rssSync.Unlock() + } else { + // otherwise update the existing feed + fd.Items = []*rss.Item{} + fd.Unread = 0 + if err := fd.Update(); err != nil { + return fmt.Errorf("Error updating address %v: %v", url, err) + } } + domain := getDomain(url) + // range over the feed and save the items for _, item := range fd.Items { id := fmt.Sprintf("%x", md5.Sum([]byte(item.ID))) - err = e.entries.Create(feeds.Entry{ + err := e.entries.Create(feeds.Entry{ Id: id, Url: item.Link, Title: item.Title, diff --git a/feeds/handler/feeds.go b/feeds/handler/feeds.go index 1e27c08..ada148d 100644 --- a/feeds/handler/feeds.go +++ b/feeds/handler/feeds.go @@ -71,7 +71,19 @@ func (e *Feeds) crawl() { } func (e *Feeds) Add(ctx context.Context, req *feeds.AddRequest, rsp *feeds.AddResponse) error { - log.Info("Received Feeds.New request") + log.Info("Received Feeds.Add request") + + if len(req.Name) == 0 { + return errors.BadRequest("feeds.add", "require name") + } + + rssSync.RLock() + defer rssSync.RUnlock() + + // check if the feed already exists + if _, ok := rssFeeds[req.Name]; ok { + return errors.BadRequest("feeds.add", "%s already exists", req.Name) + } f := feeds.Feed{ Name: req.Name,