add rss service (#117)

* add rss service

* git ignore rss

* update readme

* go fmt

* maintain map by url

* make rss service multi-tenant

rss/handler/crawl.go

@@ -0,0 +1,109 @@
+package handler
+
+import (
+	"crypto/md5"
+	"fmt"
+	"net/url"
+	"sync"
+	"time"
+
+	"github.com/SlyMarbo/rss"
+	log "github.com/micro/micro/v3/service/logger"
+	"github.com/micro/services/rss/parser"
+	pb "github.com/micro/services/rss/proto"
+)
+
+var (
+	rssSync  sync.RWMutex
+	rssFeeds = map[string]*rss.Feed{}
+)
+
+func (e *Rss) fetchAll() {
+	fs := []*pb.Feed{}
+	err := e.feeds.Read(e.feedsNameIndex.ToQuery(nil), &fs)
+	if err != nil {
+		log.Errorf("Error listing pb: %v", err)
+		return
+	}
+	if len(fs) == 0 {
+		log.Infof("No pb to fetch")
+		return
+	}
+	for _, feed := range fs {
+		err = e.fetch(feed)
+		if err != nil {
+			log.Errorf("Error saving post: %v", err)
+		}
+	}
+}
+
+func (e *Rss) fetch(f *pb.Feed) error {
+	url := f.Url
+	log.Infof("Fetching address %v", url)
+
+	// see if there's an existing rss feed
+	rssSync.RLock()
+	fd, ok := rssFeeds[f.Url]
+	rssSync.RUnlock()
+
+	if !ok {
+		// create a new one if it doesn't exist
+		var err error
+		fd, err = rss.Fetch(f.Url)
+		if err != nil {
+			return fmt.Errorf("Error fetching address %v: %v", url, err)
+		}
+		// save the feed
+		rssSync.Lock()
+		rssFeeds[f.Url] = fd
+		rssSync.Unlock()
+	} else {
+		// otherwise update the existing feed
+		fd.Items = []*rss.Item{}
+		fd.Unread = 0
+		if err := fd.Update(); err != nil {
+			return fmt.Errorf("Error updating address %v: %v", url, err)
+		}
+	}
+
+	// set the refresh time
+	fd.Refresh = time.Now()
+	domain := getDomain(url)
+
+	// range over the feed and save the items
+	for _, item := range fd.Items {
+		id := fmt.Sprintf("%x", md5.Sum([]byte(item.ID)))
+
+		// check if content exists
+		content := item.Content
+
+		// if we have a parser which returns content use it
+		// e.g cnbc
+		c, err := parser.Parse(item.Link)
+		if err == nil && len(c) > 0 {
+			content = c
+		}
+
+		err = e.entries.Create(pb.Entry{
+			Id:       id,
+			Title:    item.Title,
+			Summary:  item.Summary,
+			Url:      item.Link,
+			Domain:   domain,
+			Content:  content,
+			Date:     item.Date.Unix(),
+			Category: f.Category,
+		})
+		if err != nil {
+			return fmt.Errorf("Error saving item: %v", err)
+		}
+
+	}
+
+	return nil
+}
+
+func getDomain(address string) string {
+	uri, _ := url.Parse(address)
+	return uri.Host
+}